2024-11-26 11:14:29,927 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 11:14:29,940 main DEBUG Took 0.011398 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 11:14:29,941 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 11:14:29,941 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 11:14:29,942 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 11:14:29,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,951 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 11:14:29,965 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,967 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,968 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,970 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,971 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,973 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,974 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,975 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,976 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,977 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 11:14:29,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,980 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 11:14:29,982 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 11:14:29,983 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 11:14:29,986 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 11:14:29,987 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 11:14:29,989 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 11:14:29,989 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 11:14:29,999 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 11:14:30,002 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 11:14:30,003 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 11:14:30,004 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 11:14:30,004 main DEBUG createAppenders(={Console}) 2024-11-26 11:14:30,005 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 11:14:30,006 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 11:14:30,006 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 11:14:30,007 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 11:14:30,007 main DEBUG OutputStream closed 2024-11-26 11:14:30,008 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 11:14:30,008 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 11:14:30,008 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 11:14:30,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 11:14:30,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 11:14:30,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 11:14:30,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 11:14:30,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 11:14:30,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 11:14:30,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 11:14:30,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 11:14:30,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 11:14:30,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 11:14:30,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 11:14:30,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 11:14:30,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 11:14:30,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 11:14:30,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 11:14:30,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 11:14:30,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 11:14:30,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 11:14:30,112 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 11:14:30,113 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 11:14:30,113 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 11:14:30,114 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T11:14:30,335 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8 2024-11-26 11:14:30,337 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 11:14:30,338 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T11:14:30,347 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-26T11:14:30,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T11:14:30,381 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9, deleteOnExit=true 2024-11-26T11:14:30,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T11:14:30,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/test.cache.data in system properties and HBase conf 2024-11-26T11:14:30,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T11:14:30,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/hadoop.log.dir in system properties and HBase conf 2024-11-26T11:14:30,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T11:14:30,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T11:14:30,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T11:14:30,468 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T11:14:30,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T11:14:30,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T11:14:30,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T11:14:30,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T11:14:30,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T11:14:30,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T11:14:30,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T11:14:30,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T11:14:30,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T11:14:30,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T11:14:30,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/nfs.dump.dir in system properties and HBase conf 2024-11-26T11:14:30,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/java.io.tmpdir in system properties and HBase conf 2024-11-26T11:14:30,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T11:14:30,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T11:14:30,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T11:14:31,595 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T11:14:31,665 INFO [Time-limited test {}] log.Log(170): Logging initialized @2433ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T11:14:31,731 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T11:14:31,790 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T11:14:31,811 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T11:14:31,811 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T11:14:31,813 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T11:14:31,825 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T11:14:31,828 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/hadoop.log.dir/,AVAILABLE} 2024-11-26T11:14:31,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T11:14:32,000 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/java.io.tmpdir/jetty-localhost-45989-hadoop-hdfs-3_4_1-tests_jar-_-any-4470234302711123229/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T11:14:32,012 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45989} 2024-11-26T11:14:32,012 INFO [Time-limited test {}] server.Server(415): Started @2780ms 2024-11-26T11:14:32,572 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T11:14:32,579 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T11:14:32,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T11:14:32,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T11:14:32,582 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T11:14:32,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/hadoop.log.dir/,AVAILABLE} 2024-11-26T11:14:32,584 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T11:14:32,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/java.io.tmpdir/jetty-localhost-41347-hadoop-hdfs-3_4_1-tests_jar-_-any-8859228664724694738/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T11:14:32,696 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41347} 2024-11-26T11:14:32,697 INFO [Time-limited test {}] server.Server(415): Started @3465ms 2024-11-26T11:14:32,763 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T11:14:34,107 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/data/data2/current/BP-1195852732-172.17.0.3-1732619671074/current, will proceed with Du for space computation calculation, 2024-11-26T11:14:34,107 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/data/data1/current/BP-1195852732-172.17.0.3-1732619671074/current, will proceed with Du for space computation calculation, 2024-11-26T11:14:34,134 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T11:14:34,178 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd457f929ea96c157 with lease ID 0xbababfc40b68afb6: Processing first storage report for DS-49f69c94-1636-4105-b85f-0a8a6893db10 from datanode DatanodeRegistration(127.0.0.1:41003, datanodeUuid=64bbef44-b7bc-4876-9241-855d1bf347e8, infoPort=46033, infoSecurePort=0, ipcPort=46777, storageInfo=lv=-57;cid=testClusterID;nsid=1704059956;c=1732619671074) 2024-11-26T11:14:34,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd457f929ea96c157 with lease ID 0xbababfc40b68afb6: from storage DS-49f69c94-1636-4105-b85f-0a8a6893db10 node DatanodeRegistration(127.0.0.1:41003, datanodeUuid=64bbef44-b7bc-4876-9241-855d1bf347e8, infoPort=46033, infoSecurePort=0, ipcPort=46777, storageInfo=lv=-57;cid=testClusterID;nsid=1704059956;c=1732619671074), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T11:14:34,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd457f929ea96c157 with lease ID 0xbababfc40b68afb6: Processing first storage report for DS-f2ab18bb-7951-4633-ac85-48b95226c748 from datanode DatanodeRegistration(127.0.0.1:41003, datanodeUuid=64bbef44-b7bc-4876-9241-855d1bf347e8, infoPort=46033, infoSecurePort=0, ipcPort=46777, storageInfo=lv=-57;cid=testClusterID;nsid=1704059956;c=1732619671074) 2024-11-26T11:14:34,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd457f929ea96c157 with lease ID 0xbababfc40b68afb6: from storage DS-f2ab18bb-7951-4633-ac85-48b95226c748 node DatanodeRegistration(127.0.0.1:41003, datanodeUuid=64bbef44-b7bc-4876-9241-855d1bf347e8, infoPort=46033, infoSecurePort=0, ipcPort=46777, storageInfo=lv=-57;cid=testClusterID;nsid=1704059956;c=1732619671074), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T11:14:34,266 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8 2024-11-26T11:14:34,334 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/zookeeper_0, clientPort=64486, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T11:14:34,344 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64486 2024-11-26T11:14:34,354 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:34,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:34,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741825_1001 (size=7) 2024-11-26T11:14:34,960 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 with version=8 2024-11-26T11:14:34,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/hbase-staging 2024-11-26T11:14:35,057 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T11:14:35,272 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7f3a3edaecc3:0 server-side Connection retries=45 2024-11-26T11:14:35,281 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T11:14:35,282 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T11:14:35,287 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T11:14:35,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T11:14:35,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T11:14:35,426 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T11:14:35,485 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T11:14:35,494 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T11:14:35,497 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T11:14:35,527 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4702 (auto-detected) 2024-11-26T11:14:35,528 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-26T11:14:35,546 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42255 2024-11-26T11:14:35,567 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42255 connecting to ZooKeeper ensemble=127.0.0.1:64486 2024-11-26T11:14:35,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:422550x0, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T11:14:35,711 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42255-0x10176da1e780000 connected 2024-11-26T11:14:36,560 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:36,563 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:36,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T11:14:36,578 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29, hbase.cluster.distributed=false 2024-11-26T11:14:36,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T11:14:36,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42255 2024-11-26T11:14:36,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42255 2024-11-26T11:14:36,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42255 2024-11-26T11:14:36,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42255 2024-11-26T11:14:36,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42255 2024-11-26T11:14:36,698 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7f3a3edaecc3:0 server-side Connection retries=45 2024-11-26T11:14:36,699 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T11:14:36,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T11:14:36,700 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T11:14:36,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T11:14:36,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T11:14:36,703 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T11:14:36,705 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T11:14:36,706 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37499 2024-11-26T11:14:36,707 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37499 connecting to ZooKeeper ensemble=127.0.0.1:64486 2024-11-26T11:14:36,709 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:36,711 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:36,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374990x0, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T11:14:36,781 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374990x0, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T11:14:36,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37499-0x10176da1e780001 connected 2024-11-26T11:14:36,787 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T11:14:36,796 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T11:14:36,799 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T11:14:36,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T11:14:36,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37499 2024-11-26T11:14:36,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37499 2024-11-26T11:14:36,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37499 2024-11-26T11:14:36,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37499 2024-11-26T11:14:36,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37499 2024-11-26T11:14:36,824 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7f3a3edaecc3:42255 2024-11-26T11:14:36,825 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:36,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T11:14:36,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T11:14:36,863 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:36,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T11:14:36,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:36,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:36,936 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T11:14:36,938 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7f3a3edaecc3,42255,1732619675105 from backup master directory 2024-11-26T11:14:36,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T11:14:36,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:36,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T11:14:36,960 WARN [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T11:14:36,960 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:36,963 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T11:14:36,964 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T11:14:37,019 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/hbase.id] with ID: 4be5fd21-3c4c-4c45-86ec-49ce10916dea 2024-11-26T11:14:37,019 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/.tmp/hbase.id 2024-11-26T11:14:37,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741826_1002 (size=42) 2024-11-26T11:14:37,431 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/.tmp/hbase.id]:[hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/hbase.id] 2024-11-26T11:14:37,490 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T11:14:37,497 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T11:14:37,516 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-26T11:14:37,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:37,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:37,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741827_1003 (size=196) 2024-11-26T11:14:37,965 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:14:37,968 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T11:14:37,982 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:37,985 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T11:14:38,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741828_1004 (size=1189) 2024-11-26T11:14:38,438 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store 2024-11-26T11:14:38,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741829_1005 (size=34) 2024-11-26T11:14:38,863 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T11:14:38,867 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:14:38,868 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T11:14:38,868 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:14:38,869 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:14:38,870 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T11:14:38,871 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:14:38,871 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:14:38,872 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732619678868Disabling compacts and flushes for region at 1732619678868Disabling writes for close at 1732619678871 (+3 ms)Writing region close event to WAL at 1732619678871Closed at 1732619678871 2024-11-26T11:14:38,875 WARN [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/.initializing 2024-11-26T11:14:38,875 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/WALs/7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:38,883 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T11:14:38,898 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7f3a3edaecc3%2C42255%2C1732619675105, suffix=, logDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/WALs/7f3a3edaecc3,42255,1732619675105, archiveDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/oldWALs, maxLogs=10 2024-11-26T11:14:38,920 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/WALs/7f3a3edaecc3,42255,1732619675105/7f3a3edaecc3%2C42255%2C1732619675105.1732619678903, exclude list is [], retry=0 2024-11-26T11:14:38,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41003,DS-49f69c94-1636-4105-b85f-0a8a6893db10,DISK] 2024-11-26T11:14:38,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T11:14:38,969 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/WALs/7f3a3edaecc3,42255,1732619675105/7f3a3edaecc3%2C42255%2C1732619675105.1732619678903 2024-11-26T11:14:38,970 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46033:46033)] 2024-11-26T11:14:38,971 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:14:38,971 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:14:38,974 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:38,975 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,010 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T11:14:39,036 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:39,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:39,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T11:14:39,042 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:39,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:14:39,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T11:14:39,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:39,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:14:39,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T11:14:39,050 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:39,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:14:39,051 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,054 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,056 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,061 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,062 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,065 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T11:14:39,069 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T11:14:39,074 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:14:39,075 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59464977, jitterRate=-0.11390279233455658}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T11:14:39,082 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732619678986Initializing all the Stores at 1732619678988 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619678989 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619678989Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619678990 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619678990Cleaning up temporary data from old regions at 1732619679062 (+72 ms)Region opened successfully at 1732619679082 (+20 ms) 2024-11-26T11:14:39,084 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T11:14:39,119 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad6cb9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7f3a3edaecc3/172.17.0.3:0 2024-11-26T11:14:39,149 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T11:14:39,159 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T11:14:39,159 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T11:14:39,162 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T11:14:39,163 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T11:14:39,168 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T11:14:39,168 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T11:14:39,193 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T11:14:39,203 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T11:14:39,296 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T11:14:39,302 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T11:14:39,306 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T11:14:39,317 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T11:14:39,320 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T11:14:39,324 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T11:14:39,331 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T11:14:39,333 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T11:14:39,342 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T11:14:39,364 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T11:14:39,373 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T11:14:39,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T11:14:39,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T11:14:39,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:39,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:39,388 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7f3a3edaecc3,42255,1732619675105, sessionid=0x10176da1e780000, setting cluster-up flag (Was=false) 2024-11-26T11:14:39,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:39,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:39,458 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T11:14:39,462 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:39,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:39,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:39,521 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T11:14:39,525 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:39,533 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T11:14:39,597 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T11:14:39,606 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T11:14:39,611 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T11:14:39,616 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(746): ClusterId : 4be5fd21-3c4c-4c45-86ec-49ce10916dea 2024-11-26T11:14:39,618 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T11:14:39,617 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7f3a3edaecc3,42255,1732619675105 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T11:14:39,628 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T11:14:39,628 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T11:14:39,629 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7f3a3edaecc3:0, corePoolSize=5, maxPoolSize=5 2024-11-26T11:14:39,629 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7f3a3edaecc3:0, corePoolSize=5, maxPoolSize=5 2024-11-26T11:14:39,629 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7f3a3edaecc3:0, corePoolSize=5, maxPoolSize=5 2024-11-26T11:14:39,629 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7f3a3edaecc3:0, corePoolSize=5, maxPoolSize=5 2024-11-26T11:14:39,629 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7f3a3edaecc3:0, corePoolSize=10, maxPoolSize=10 2024-11-26T11:14:39,630 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,630 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7f3a3edaecc3:0, corePoolSize=2, maxPoolSize=2 2024-11-26T11:14:39,630 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,631 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732619709631 2024-11-26T11:14:39,633 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T11:14:39,634 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T11:14:39,635 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T11:14:39,636 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T11:14:39,637 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T11:14:39,638 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T11:14:39,638 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T11:14:39,638 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T11:14:39,638 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T11:14:39,639 DEBUG [RS:0;7f3a3edaecc3:37499 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d91129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7f3a3edaecc3/172.17.0.3:0 2024-11-26T11:14:39,639 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,644 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:39,644 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T11:14:39,644 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T11:14:39,646 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T11:14:39,646 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T11:14:39,648 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T11:14:39,648 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T11:14:39,652 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7f3a3edaecc3:0:becomeActiveMaster-HFileCleaner.large.0-1732619679650,5,FailOnTimeoutGroup] 2024-11-26T11:14:39,653 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7f3a3edaecc3:0:becomeActiveMaster-HFileCleaner.small.0-1732619679652,5,FailOnTimeoutGroup] 2024-11-26T11:14:39,653 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,653 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T11:14:39,654 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,655 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,655 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7f3a3edaecc3:37499 2024-11-26T11:14:39,659 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T11:14:39,659 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T11:14:39,659 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T11:14:39,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741831_1007 (size=1321) 2024-11-26T11:14:39,663 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(2659): reportForDuty to master=7f3a3edaecc3,42255,1732619675105 with port=37499, startcode=1732619676667 2024-11-26T11:14:39,677 DEBUG [RS:0;7f3a3edaecc3:37499 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T11:14:39,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56795, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T11:14:39,719 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42255 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:39,721 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42255 {}] master.ServerManager(517): Registering regionserver=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:39,733 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:14:39,733 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41651 2024-11-26T11:14:39,733 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T11:14:39,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T11:14:39,749 DEBUG [RS:0;7f3a3edaecc3:37499 {}] zookeeper.ZKUtil(111): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:39,749 WARN [RS:0;7f3a3edaecc3:37499 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T11:14:39,750 INFO [RS:0;7f3a3edaecc3:37499 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T11:14:39,750 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:39,752 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7f3a3edaecc3,37499,1732619676667] 2024-11-26T11:14:39,777 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T11:14:39,793 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T11:14:39,798 INFO [RS:0;7f3a3edaecc3:37499 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T11:14:39,798 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,800 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T11:14:39,805 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T11:14:39,807 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,807 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,808 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,808 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,808 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,808 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,808 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7f3a3edaecc3:0, corePoolSize=2, maxPoolSize=2 2024-11-26T11:14:39,809 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,809 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,809 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,809 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,809 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,809 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7f3a3edaecc3:0, corePoolSize=1, maxPoolSize=1 2024-11-26T11:14:39,810 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7f3a3edaecc3:0, corePoolSize=3, maxPoolSize=3 2024-11-26T11:14:39,810 DEBUG [RS:0;7f3a3edaecc3:37499 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0, corePoolSize=3, maxPoolSize=3 2024-11-26T11:14:39,811 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,811 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,811 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,811 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,812 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,812 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,37499,1732619676667-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T11:14:39,832 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T11:14:39,833 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,37499,1732619676667-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,834 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,834 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.Replication(171): 7f3a3edaecc3,37499,1732619676667 started 2024-11-26T11:14:39,851 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:39,852 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1482): Serving as 7f3a3edaecc3,37499,1732619676667, RpcServer on 7f3a3edaecc3/172.17.0.3:37499, sessionid=0x10176da1e780001 2024-11-26T11:14:39,852 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T11:14:39,853 DEBUG [RS:0;7f3a3edaecc3:37499 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:39,853 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7f3a3edaecc3,37499,1732619676667' 2024-11-26T11:14:39,853 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T11:14:39,854 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T11:14:39,855 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T11:14:39,855 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T11:14:39,855 DEBUG [RS:0;7f3a3edaecc3:37499 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:39,855 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7f3a3edaecc3,37499,1732619676667' 2024-11-26T11:14:39,856 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T11:14:39,856 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T11:14:39,857 DEBUG [RS:0;7f3a3edaecc3:37499 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T11:14:39,857 INFO [RS:0;7f3a3edaecc3:37499 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T11:14:39,857 INFO [RS:0;7f3a3edaecc3:37499 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T11:14:39,965 INFO [RS:0;7f3a3edaecc3:37499 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T11:14:39,969 INFO [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7f3a3edaecc3%2C37499%2C1732619676667, suffix=, logDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667, archiveDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/oldWALs, maxLogs=32 2024-11-26T11:14:39,984 DEBUG [RS:0;7f3a3edaecc3:37499 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667/7f3a3edaecc3%2C37499%2C1732619676667.1732619679972, exclude list is [], retry=0 2024-11-26T11:14:39,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41003,DS-49f69c94-1636-4105-b85f-0a8a6893db10,DISK] 2024-11-26T11:14:39,993 INFO [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667/7f3a3edaecc3%2C37499%2C1732619676667.1732619679972 2024-11-26T11:14:39,994 DEBUG [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46033:46033)] 2024-11-26T11:14:40,065 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T11:14:40,066 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:14:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741833_1009 (size=32) 2024-11-26T11:14:40,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:14:40,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T11:14:40,496 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T11:14:40,496 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:40,497 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:40,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T11:14:40,500 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T11:14:40,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:40,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:40,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T11:14:40,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T11:14:40,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:40,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:40,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T11:14:40,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T11:14:40,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:40,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:40,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T11:14:40,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740 2024-11-26T11:14:40,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740 2024-11-26T11:14:40,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T11:14:40,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T11:14:40,514 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:14:40,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T11:14:40,519 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:14:40,520 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64859166, jitterRate=-0.033523112535476685}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:14:40,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732619680488Initializing all the Stores at 1732619680493 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619680493Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619680493Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619680493Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619680493Cleaning up temporary data from old regions at 1732619680513 (+20 ms)Region opened successfully at 1732619680523 (+10 ms) 2024-11-26T11:14:40,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T11:14:40,523 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T11:14:40,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T11:14:40,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T11:14:40,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T11:14:40,525 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T11:14:40,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732619680523Disabling compacts and flushes for region at 1732619680523Disabling writes for close at 1732619680523Writing region close event to WAL at 1732619680524 (+1 ms)Closed at 1732619680525 (+1 ms) 2024-11-26T11:14:40,528 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T11:14:40,528 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T11:14:40,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T11:14:40,541 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T11:14:40,544 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T11:14:40,701 DEBUG [7f3a3edaecc3:42255 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T11:14:40,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:40,721 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7f3a3edaecc3,37499,1732619676667, state=OPENING 2024-11-26T11:14:40,769 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T11:14:40,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:40,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:14:40,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T11:14:40,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T11:14:40,785 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T11:14:40,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:14:40,968 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T11:14:40,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40435, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T11:14:40,982 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T11:14:40,983 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T11:14:40,983 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T11:14:40,986 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7f3a3edaecc3%2C37499%2C1732619676667.meta, suffix=.meta, logDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667, archiveDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/oldWALs, maxLogs=32 2024-11-26T11:14:41,001 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667/7f3a3edaecc3%2C37499%2C1732619676667.meta.1732619680988.meta, exclude list is [], retry=0 2024-11-26T11:14:41,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41003,DS-49f69c94-1636-4105-b85f-0a8a6893db10,DISK] 2024-11-26T11:14:41,008 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/WALs/7f3a3edaecc3,37499,1732619676667/7f3a3edaecc3%2C37499%2C1732619676667.meta.1732619680988.meta 2024-11-26T11:14:41,008 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46033:46033)] 2024-11-26T11:14:41,009 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:14:41,010 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T11:14:41,013 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T11:14:41,017 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T11:14:41,021 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T11:14:41,022 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:14:41,022 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T11:14:41,022 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T11:14:41,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T11:14:41,027 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T11:14:41,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:41,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:41,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T11:14:41,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T11:14:41,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:41,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:41,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T11:14:41,033 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T11:14:41,033 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:41,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:41,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T11:14:41,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T11:14:41,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:41,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T11:14:41,037 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T11:14:41,039 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740 2024-11-26T11:14:41,041 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740 2024-11-26T11:14:41,044 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T11:14:41,044 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T11:14:41,045 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:14:41,048 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T11:14:41,050 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67068903, jitterRate=-5.954653024673462E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:14:41,050 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T11:14:41,051 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732619681023Writing region info on filesystem at 1732619681023Initializing all the Stores at 1732619681025 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619681025Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619681025Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619681025Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732619681025Cleaning up temporary data from old regions at 1732619681044 (+19 ms)Running coprocessor post-open hooks at 1732619681050 (+6 ms)Region opened successfully at 1732619681051 (+1 ms) 2024-11-26T11:14:41,058 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732619680961 2024-11-26T11:14:41,068 DEBUG [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T11:14:41,069 INFO [RS_OPEN_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T11:14:41,070 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:41,073 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7f3a3edaecc3,37499,1732619676667, state=OPEN 2024-11-26T11:14:41,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T11:14:41,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T11:14:41,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T11:14:41,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T11:14:41,113 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:41,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T11:14:41,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7f3a3edaecc3,37499,1732619676667 in 326 msec 2024-11-26T11:14:41,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T11:14:41,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 591 msec 2024-11-26T11:14:41,133 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T11:14:41,133 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T11:14:41,153 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:41,154 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:41,174 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:41,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43233, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:41,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6350 sec 2024-11-26T11:14:41,196 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732619681196, completionTime=-1 2024-11-26T11:14:41,199 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T11:14:41,199 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T11:14:41,223 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T11:14:41,223 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732619741223 2024-11-26T11:14:41,223 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732619801223 2024-11-26T11:14:41,223 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-26T11:14:41,225 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,42255,1732619675105-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,225 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,42255,1732619675105-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,226 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,42255,1732619675105-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,227 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7f3a3edaecc3:42255, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,227 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,228 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,234 DEBUG [master/7f3a3edaecc3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T11:14:41,254 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.293sec 2024-11-26T11:14:41,255 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T11:14:41,256 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T11:14:41,257 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T11:14:41,258 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T11:14:41,258 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T11:14:41,259 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,42255,1732619675105-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T11:14:41,259 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,42255,1732619675105-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T11:14:41,268 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T11:14:41,269 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T11:14:41,270 INFO [master/7f3a3edaecc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7f3a3edaecc3,42255,1732619675105-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T11:14:41,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:41,333 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T11:14:41,333 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T11:14:41,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:41,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:41,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:41,356 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:41,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:41,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:41,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:41,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:41,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=3495 2024-11-26T11:14:41,408 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:14:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f523d87 2024-11-26T11:14:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:14:41,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:14:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:14:41,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:14:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T11:14:41,454 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T11:14:41,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T11:14:41,460 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:41,462 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T11:14:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T11:14:41,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741835_1011 (size=1061) 2024-11-26T11:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T11:14:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T11:14:41,884 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:14:41,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741836_1012 (size=53) 2024-11-26T11:14:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T11:14:42,295 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:14:42,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 77c23a8be60706f6017524015b3171ad, disabling compactions & flushes 2024-11-26T11:14:42,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. after waiting 0 ms 2024-11-26T11:14:42,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,297 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 77c23a8be60706f6017524015b3171ad: Waiting for close lock at 1732619682296Disabling compacts and flushes for region at 1732619682296Disabling writes for close at 1732619682296Writing region close event to WAL at 1732619682296Closed at 1732619682296 2024-11-26T11:14:42,301 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T11:14:42,310 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732619682301"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732619682301"}]},"ts":"1732619682301"} 2024-11-26T11:14:42,314 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T11:14:42,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T11:14:42,319 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619682317"}]},"ts":"1732619682317"} 2024-11-26T11:14:42,327 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T11:14:42,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, ASSIGN}] 2024-11-26T11:14:42,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, ASSIGN 2024-11-26T11:14:42,333 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, ASSIGN; state=OFFLINE, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=false 2024-11-26T11:14:42,486 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=77c23a8be60706f6017524015b3171ad, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:42,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, ASSIGN because future has completed 2024-11-26T11:14:42,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:14:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T11:14:42,661 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,661 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:14:42,662 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,662 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:14:42,662 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,662 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,665 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,669 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:14:42,669 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77c23a8be60706f6017524015b3171ad columnFamilyName A 2024-11-26T11:14:42,669 DEBUG [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:42,670 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.HStore(327): Store=77c23a8be60706f6017524015b3171ad/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:14:42,671 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,673 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:14:42,673 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77c23a8be60706f6017524015b3171ad columnFamilyName B 2024-11-26T11:14:42,673 DEBUG [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:42,674 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.HStore(327): Store=77c23a8be60706f6017524015b3171ad/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:14:42,674 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,676 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:14:42,677 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 77c23a8be60706f6017524015b3171ad columnFamilyName C 2024-11-26T11:14:42,677 DEBUG [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:14:42,678 INFO [StoreOpener-77c23a8be60706f6017524015b3171ad-1 {}] regionserver.HStore(327): Store=77c23a8be60706f6017524015b3171ad/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:14:42,678 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,679 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,680 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,681 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,683 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,683 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,683 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:14:42,686 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,689 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:14:42,690 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 77c23a8be60706f6017524015b3171ad; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72177247, jitterRate=0.07552479207515717}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:14:42,690 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:42,691 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 77c23a8be60706f6017524015b3171ad: Running coprocessor pre-open hook at 1732619682662Writing region info on filesystem at 1732619682662Initializing all the Stores at 1732619682664 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619682664Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619682664Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619682664Cleaning up temporary data from old regions at 1732619682683 (+19 ms)Running coprocessor post-open hooks at 1732619682690 (+7 ms)Region opened successfully at 1732619682691 (+1 ms) 2024-11-26T11:14:42,693 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., pid=6, masterSystemTime=1732619682651 2024-11-26T11:14:42,697 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,697 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:42,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=77c23a8be60706f6017524015b3171ad, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:42,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:14:42,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T11:14:42,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 in 209 msec 2024-11-26T11:14:42,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T11:14:42,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, ASSIGN in 380 msec 2024-11-26T11:14:42,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T11:14:42,717 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619682717"}]},"ts":"1732619682717"} 2024-11-26T11:14:42,720 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T11:14:42,722 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T11:14:42,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2770 sec 2024-11-26T11:14:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T11:14:43,628 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:14:43,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6a9f59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708c62f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,647 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:14:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:14:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e704312 2024-11-26T11:14:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:14:43,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:14:43,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,678 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,691 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a081598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,702 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5904dd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,705 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,708 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,711 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee9f900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,714 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4e5c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1118bf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:14:43,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:14:43,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:14:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:14:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:14:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d56454e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:14:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:14:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:14:43,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:14:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6514b0c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:14:43,730 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,730 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,730 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,730 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,730 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,730 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,730 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,738 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,741 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:14:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:14:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:14:43,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:14:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:14:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:14:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T11:14:43,762 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:14:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:43,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:14:43,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:14:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:43,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T11:14:43,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:43,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:43,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:43,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:43,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:43,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:43,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8477a500078b4d3faa051bb78388210a is 50, key is test_row_0/A:col10/1732619683830/Put/seqid=0 2024-11-26T11:14:43,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:43,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:43,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:43,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:43,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:43,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619743972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:43,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741837_1013 (size=19021) 2024-11-26T11:14:43,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619743979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619743980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619743986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619743986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619744134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619744135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619744135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619744136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619744136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:44,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619744354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619744355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619744356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619744356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619744357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:44,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8477a500078b4d3faa051bb78388210a 2024-11-26T11:14:44,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:44,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e9b6b4352ddc4fcbb04b62499b4e3325 is 50, key is test_row_0/B:col10/1732619683830/Put/seqid=0 2024-11-26T11:14:44,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741838_1014 (size=12001) 2024-11-26T11:14:44,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e9b6b4352ddc4fcbb04b62499b4e3325 2024-11-26T11:14:44,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd is 50, key is test_row_0/C:col10/1732619683830/Put/seqid=0 2024-11-26T11:14:44,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741839_1015 (size=12001) 2024-11-26T11:14:44,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd 2024-11-26T11:14:44,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8477a500078b4d3faa051bb78388210a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8477a500078b4d3faa051bb78388210a 2024-11-26T11:14:44,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8477a500078b4d3faa051bb78388210a, entries=300, sequenceid=14, filesize=18.6 K 2024-11-26T11:14:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-26T11:14:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e9b6b4352ddc4fcbb04b62499b4e3325 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e9b6b4352ddc4fcbb04b62499b4e3325 2024-11-26T11:14:44,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e9b6b4352ddc4fcbb04b62499b4e3325, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T11:14:44,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd 2024-11-26T11:14:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619744666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619744666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619744666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:44,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T11:14:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:44,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 77c23a8be60706f6017524015b3171ad in 877ms, sequenceid=14, compaction requested=false 2024-11-26T11:14:44,720 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T11:14:44,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:44,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T11:14:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:44,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619683938, max timestamp=1732619684710Num uniques 859; 2024-11-26T11:14:44,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:44,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619683938, max timestamp=1732619684710Num uniques 859; 2024-11-26T11:14:44,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:44,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:44,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/7b88cc2bf5d94c9088525f50ad7fc96c is 50, key is test_row_0/A:col10/1732619683977/Put/seqid=0 2024-11-26T11:14:44,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741840_1016 (size=16681) 2024-11-26T11:14:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:44,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:44,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:44,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619745034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619745052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:45,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:45,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619745174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619745175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/7b88cc2bf5d94c9088525f50ad7fc96c 2024-11-26T11:14:45,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619745215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619745215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619745223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/caefcb08f9af423bb8726465bfc677db is 50, key is test_row_0/B:col10/1732619683977/Put/seqid=0 2024-11-26T11:14:45,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:45,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:45,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741841_1017 (size=12001) 2024-11-26T11:14:45,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/caefcb08f9af423bb8726465bfc677db 2024-11-26T11:14:45,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/22d50c5c70d8457890141427d77a8b73 is 50, key is test_row_0/C:col10/1732619683977/Put/seqid=0 2024-11-26T11:14:45,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741842_1018 (size=12001) 2024-11-26T11:14:45,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/22d50c5c70d8457890141427d77a8b73 2024-11-26T11:14:45,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/7b88cc2bf5d94c9088525f50ad7fc96c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7b88cc2bf5d94c9088525f50ad7fc96c 2024-11-26T11:14:45,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7b88cc2bf5d94c9088525f50ad7fc96c, entries=250, sequenceid=36, filesize=16.3 K 2024-11-26T11:14:45,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/caefcb08f9af423bb8726465bfc677db as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/caefcb08f9af423bb8726465bfc677db 2024-11-26T11:14:45,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/caefcb08f9af423bb8726465bfc677db, entries=150, sequenceid=36, filesize=11.7 K 2024-11-26T11:14:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/22d50c5c70d8457890141427d77a8b73 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/22d50c5c70d8457890141427d77a8b73 2024-11-26T11:14:45,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/22d50c5c70d8457890141427d77a8b73, entries=150, sequenceid=36, filesize=11.7 K 2024-11-26T11:14:45,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 77c23a8be60706f6017524015b3171ad in 669ms, sequenceid=36, compaction requested=false 2024-11-26T11:14:45,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:45,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T11:14:45,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:45,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:14:45,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619684994, max timestamp=1732619685408Num uniques 859; 2024-11-26T11:14:45,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:45,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:45,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/48fa92fb55ae42918a1db6fb221b1a74 is 50, key is test_row_0/A:col10/1732619685408/Put/seqid=0 2024-11-26T11:14:45,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741843_1019 (size=14341) 2024-11-26T11:14:45,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T11:14:45,483 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T11:14:45,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T11:14:45,491 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T11:14:45,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T11:14:45,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T11:14:45,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T11:14:45,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T11:14:45,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619745509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619745519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619745633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619745636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,771 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T11:14:45,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619745844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619745844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:45,853 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/48fa92fb55ae42918a1db6fb221b1a74 2024-11-26T11:14:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/13d66241bb214075a57871f408731223 is 50, key is test_row_0/B:col10/1732619685408/Put/seqid=0 2024-11-26T11:14:45,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741844_1020 (size=12001) 2024-11-26T11:14:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619746153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619746153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619746253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619746254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619746254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,261 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,262 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,261 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,313 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/13d66241bb214075a57871f408731223 2024-11-26T11:14:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/d03c403d85524888a5d78a601f471036 is 50, key is test_row_0/C:col10/1732619685408/Put/seqid=0 2024-11-26T11:14:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741845_1021 (size=12001) 2024-11-26T11:14:46,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/d03c403d85524888a5d78a601f471036 2024-11-26T11:14:46,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/48fa92fb55ae42918a1db6fb221b1a74 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/48fa92fb55ae42918a1db6fb221b1a74 2024-11-26T11:14:46,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/48fa92fb55ae42918a1db6fb221b1a74, entries=200, sequenceid=57, filesize=14.0 K 2024-11-26T11:14:46,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/13d66241bb214075a57871f408731223 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/13d66241bb214075a57871f408731223 2024-11-26T11:14:46,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/13d66241bb214075a57871f408731223, entries=150, sequenceid=57, filesize=11.7 K 2024-11-26T11:14:46,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/d03c403d85524888a5d78a601f471036 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d03c403d85524888a5d78a601f471036 2024-11-26T11:14:46,433 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d03c403d85524888a5d78a601f471036, entries=150, sequenceid=57, filesize=11.7 K 2024-11-26T11:14:46,435 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 77c23a8be60706f6017524015b3171ad in 1026ms, sequenceid=57, compaction requested=true 2024-11-26T11:14:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T11:14:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T11:14:46,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T11:14:46,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6700 sec 2024-11-26T11:14:46,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.6890 sec 2024-11-26T11:14:46,547 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:14:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:46,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:14:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:46,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:46,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:46,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/df31f2af58fc4a8fa29b1a7976a3d83a is 50, key is test_row_0/A:col10/1732619685496/Put/seqid=0 2024-11-26T11:14:46,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741846_1022 (size=14341) 2024-11-26T11:14:46,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619746729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619746735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619746843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619746844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:47,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619747063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619747063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:47,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/df31f2af58fc4a8fa29b1a7976a3d83a 2024-11-26T11:14:47,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/adf800f15980455d9b85ea5a89f6b66c is 50, key is test_row_0/B:col10/1732619685496/Put/seqid=0 2024-11-26T11:14:47,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741847_1023 (size=12001) 2024-11-26T11:14:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:47,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619747373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:47,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/adf800f15980455d9b85ea5a89f6b66c 2024-11-26T11:14:47,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0d58730d21ca464ab15d6a110f41a634 is 50, key is test_row_0/C:col10/1732619685496/Put/seqid=0 2024-11-26T11:14:47,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741848_1024 (size=12001) 2024-11-26T11:14:47,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0d58730d21ca464ab15d6a110f41a634 2024-11-26T11:14:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/df31f2af58fc4a8fa29b1a7976a3d83a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/df31f2af58fc4a8fa29b1a7976a3d83a 2024-11-26T11:14:47,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/df31f2af58fc4a8fa29b1a7976a3d83a, entries=200, sequenceid=74, filesize=14.0 K 2024-11-26T11:14:47,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/adf800f15980455d9b85ea5a89f6b66c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/adf800f15980455d9b85ea5a89f6b66c 2024-11-26T11:14:47,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/adf800f15980455d9b85ea5a89f6b66c, entries=150, sequenceid=74, filesize=11.7 K 2024-11-26T11:14:47,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0d58730d21ca464ab15d6a110f41a634 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0d58730d21ca464ab15d6a110f41a634 2024-11-26T11:14:47,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0d58730d21ca464ab15d6a110f41a634, entries=150, sequenceid=74, filesize=11.7 K 2024-11-26T11:14:47,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 77c23a8be60706f6017524015b3171ad in 994ms, sequenceid=74, compaction requested=true 2024-11-26T11:14:47,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:47,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:14:47,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:47,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:14:47,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:47,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:14:47,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T11:14:47,668 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:47,669 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:47,677 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:47,679 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:14:47,680 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:47,680 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 64384 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:47,680 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:14:47,680 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:47,681 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e9b6b4352ddc4fcbb04b62499b4e3325, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/caefcb08f9af423bb8726465bfc677db, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/13d66241bb214075a57871f408731223, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/adf800f15980455d9b85ea5a89f6b66c] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=46.9 K 2024-11-26T11:14:47,681 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8477a500078b4d3faa051bb78388210a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7b88cc2bf5d94c9088525f50ad7fc96c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/48fa92fb55ae42918a1db6fb221b1a74, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/df31f2af58fc4a8fa29b1a7976a3d83a] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=62.9 K 2024-11-26T11:14:47,682 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b6b4352ddc4fcbb04b62499b4e3325, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619683812 2024-11-26T11:14:47,683 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8477a500078b4d3faa051bb78388210a, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619683796 2024-11-26T11:14:47,684 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting caefcb08f9af423bb8726465bfc677db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732619683977 2024-11-26T11:14:47,684 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b88cc2bf5d94c9088525f50ad7fc96c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732619683938 2024-11-26T11:14:47,685 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d66241bb214075a57871f408731223, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619684994 2024-11-26T11:14:47,689 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48fa92fb55ae42918a1db6fb221b1a74, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619684994 2024-11-26T11:14:47,690 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting adf800f15980455d9b85ea5a89f6b66c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732619685472 2024-11-26T11:14:47,691 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting df31f2af58fc4a8fa29b1a7976a3d83a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732619685472 2024-11-26T11:14:47,745 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#13 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:47,746 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e09b6c7aeaa14353b91eac0bcd9e9a61 is 50, key is test_row_0/B:col10/1732619685496/Put/seqid=0 2024-11-26T11:14:47,747 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#12 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:47,748 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/7dd0f2ef3a09410394398362f8984226 is 50, key is test_row_0/A:col10/1732619685496/Put/seqid=0 2024-11-26T11:14:47,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741849_1025 (size=12139) 2024-11-26T11:14:47,787 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e09b6c7aeaa14353b91eac0bcd9e9a61 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e09b6c7aeaa14353b91eac0bcd9e9a61 2024-11-26T11:14:47,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741850_1026 (size=12139) 2024-11-26T11:14:47,818 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into e09b6c7aeaa14353b91eac0bcd9e9a61(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:47,818 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:47,821 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=12, startTime=1732619687665; duration=0sec 2024-11-26T11:14:47,821 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:47,821 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:14:47,822 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:47,824 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:47,825 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:14:47,825 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:47,825 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/22d50c5c70d8457890141427d77a8b73, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d03c403d85524888a5d78a601f471036, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0d58730d21ca464ab15d6a110f41a634] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=46.9 K 2024-11-26T11:14:47,827 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b0cb6ee2c44e8cb6e7f10a7f4472bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619683812 2024-11-26T11:14:47,827 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d50c5c70d8457890141427d77a8b73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732619683977 2024-11-26T11:14:47,828 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d03c403d85524888a5d78a601f471036, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619684994 2024-11-26T11:14:47,829 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d58730d21ca464ab15d6a110f41a634, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732619685472 2024-11-26T11:14:47,875 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#14 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:47,876 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/933c0d1c4625448eb7d5c2bc4738e71f is 50, key is test_row_0/C:col10/1732619685496/Put/seqid=0 2024-11-26T11:14:47,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741851_1027 (size=12139) 2024-11-26T11:14:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:47,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:14:47,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:47,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:47,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:47,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:47,907 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/933c0d1c4625448eb7d5c2bc4738e71f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/933c0d1c4625448eb7d5c2bc4738e71f 2024-11-26T11:14:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T11:14:47,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:14:47,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:14:47,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/389ab5ed5abf4a55baef98b04110cd5a is 50, key is test_row_0/A:col10/1732619687894/Put/seqid=0 2024-11-26T11:14:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T11:14:47,931 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:14:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:47,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:14:47,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:14:47,940 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 933c0d1c4625448eb7d5c2bc4738e71f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:47,941 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:47,941 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=12, startTime=1732619687665; duration=0sec 2024-11-26T11:14:47,941 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:47,941 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:14:47,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741852_1028 (size=12001) 2024-11-26T11:14:47,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/389ab5ed5abf4a55baef98b04110cd5a 2024-11-26T11:14:47,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/04f9039c07f14d6f95f5d2dfd1305187 is 50, key is test_row_0/B:col10/1732619687894/Put/seqid=0 2024-11-26T11:14:48,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741853_1029 (size=12001) 2024-11-26T11:14:48,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619748017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619748020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:48,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619748133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619748134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,208 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/7dd0f2ef3a09410394398362f8984226 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7dd0f2ef3a09410394398362f8984226 2024-11-26T11:14:48,222 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into 7dd0f2ef3a09410394398362f8984226(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:48,223 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:48,223 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=12, startTime=1732619687662; duration=0sec 2024-11-26T11:14:48,223 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:48,223 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:14:48,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:48,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:48,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:48,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619748273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619748273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,278 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,279 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619748302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,305 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4334 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619748343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619748343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:48,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:48,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/04f9039c07f14d6f95f5d2dfd1305187 2024-11-26T11:14:48,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/523f1408a496401e993e4c7afeb3d1d0 is 50, key is test_row_0/C:col10/1732619687894/Put/seqid=0 2024-11-26T11:14:48,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741854_1030 (size=12001) 2024-11-26T11:14:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:48,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619748653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619748663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:48,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/523f1408a496401e993e4c7afeb3d1d0 2024-11-26T11:14:48,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/389ab5ed5abf4a55baef98b04110cd5a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/389ab5ed5abf4a55baef98b04110cd5a 2024-11-26T11:14:48,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:48,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:48,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/389ab5ed5abf4a55baef98b04110cd5a, entries=150, sequenceid=91, filesize=11.7 K 2024-11-26T11:14:48,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/04f9039c07f14d6f95f5d2dfd1305187 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/04f9039c07f14d6f95f5d2dfd1305187 2024-11-26T11:14:48,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/04f9039c07f14d6f95f5d2dfd1305187, entries=150, sequenceid=91, filesize=11.7 K 2024-11-26T11:14:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/523f1408a496401e993e4c7afeb3d1d0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/523f1408a496401e993e4c7afeb3d1d0 2024-11-26T11:14:48,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/523f1408a496401e993e4c7afeb3d1d0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-26T11:14:48,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 77c23a8be60706f6017524015b3171ad in 1070ms, sequenceid=91, compaction requested=false 2024-11-26T11:14:48,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:49,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T11:14:49,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:49,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:14:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:49,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/44a205c267de411c884a5aa6441abbab is 50, key is test_row_0/A:col10/1732619687951/Put/seqid=0 2024-11-26T11:14:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:49,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741855_1031 (size=12001) 2024-11-26T11:14:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:49,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619749212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619749214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619749323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619749323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/44a205c267de411c884a5aa6441abbab 2024-11-26T11:14:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/71b902c88783466497b06d4df49e6a37 is 50, key is test_row_0/B:col10/1732619687951/Put/seqid=0 2024-11-26T11:14:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619749533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619749533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741856_1032 (size=12001) 2024-11-26T11:14:49,557 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/71b902c88783466497b06d4df49e6a37 2024-11-26T11:14:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/d5c284724607479b9fc4a3976caa93e5 is 50, key is test_row_0/C:col10/1732619687951/Put/seqid=0 2024-11-26T11:14:49,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741857_1033 (size=12001) 2024-11-26T11:14:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619749843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619749843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/d5c284724607479b9fc4a3976caa93e5 2024-11-26T11:14:50,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/44a205c267de411c884a5aa6441abbab as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/44a205c267de411c884a5aa6441abbab 2024-11-26T11:14:50,041 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/44a205c267de411c884a5aa6441abbab, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T11:14:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/71b902c88783466497b06d4df49e6a37 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/71b902c88783466497b06d4df49e6a37 2024-11-26T11:14:50,065 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/71b902c88783466497b06d4df49e6a37, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T11:14:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/d5c284724607479b9fc4a3976caa93e5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d5c284724607479b9fc4a3976caa93e5 2024-11-26T11:14:50,080 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d5c284724607479b9fc4a3976caa93e5, entries=150, sequenceid=109, filesize=11.7 K 2024-11-26T11:14:50,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 77c23a8be60706f6017524015b3171ad in 1044ms, sequenceid=109, compaction requested=true 2024-11-26T11:14:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T11:14:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T11:14:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:50,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T11:14:50,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1490 sec 2024-11-26T11:14:50,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.1670 sec 2024-11-26T11:14:50,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:50,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:14:50,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:50,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:50,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:50,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:50,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:50,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:50,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/3e3685e7d62b4a6897b53983c3e84c4b is 50, key is test_row_0/A:col10/1732619690358/Put/seqid=0 2024-11-26T11:14:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741858_1034 (size=16681) 2024-11-26T11:14:50,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:50,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619750396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619750399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619750513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619750513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619750733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619750733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:50,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/3e3685e7d62b4a6897b53983c3e84c4b 2024-11-26T11:14:50,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/1e0366b5b77c43c09088ad199203d823 is 50, key is test_row_0/B:col10/1732619690358/Put/seqid=0 2024-11-26T11:14:50,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741859_1035 (size=12001) 2024-11-26T11:14:50,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/1e0366b5b77c43c09088ad199203d823 2024-11-26T11:14:50,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/4e9ba4da81264ea1a414ea761d20c492 is 50, key is test_row_0/C:col10/1732619690358/Put/seqid=0 2024-11-26T11:14:50,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741860_1036 (size=12001) 2024-11-26T11:14:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619751043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619751043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/4e9ba4da81264ea1a414ea761d20c492 2024-11-26T11:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/3e3685e7d62b4a6897b53983c3e84c4b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/3e3685e7d62b4a6897b53983c3e84c4b 2024-11-26T11:14:51,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/3e3685e7d62b4a6897b53983c3e84c4b, entries=250, sequenceid=127, filesize=16.3 K 2024-11-26T11:14:51,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/1e0366b5b77c43c09088ad199203d823 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/1e0366b5b77c43c09088ad199203d823 2024-11-26T11:14:51,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/1e0366b5b77c43c09088ad199203d823, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T11:14:51,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/4e9ba4da81264ea1a414ea761d20c492 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/4e9ba4da81264ea1a414ea761d20c492 2024-11-26T11:14:51,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/4e9ba4da81264ea1a414ea761d20c492, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T11:14:51,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 77c23a8be60706f6017524015b3171ad in 907ms, sequenceid=127, compaction requested=true 2024-11-26T11:14:51,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:51,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:14:51,268 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:51,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:51,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:14:51,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:51,268 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:51,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:14:51,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:51,270 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52822 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:51,270 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:14:51,270 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:51,271 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7dd0f2ef3a09410394398362f8984226, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/389ab5ed5abf4a55baef98b04110cd5a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/44a205c267de411c884a5aa6441abbab, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/3e3685e7d62b4a6897b53983c3e84c4b] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=51.6 K 2024-11-26T11:14:51,271 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:51,271 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:14:51,271 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:51,271 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e09b6c7aeaa14353b91eac0bcd9e9a61, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/04f9039c07f14d6f95f5d2dfd1305187, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/71b902c88783466497b06d4df49e6a37, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/1e0366b5b77c43c09088ad199203d823] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=47.0 K 2024-11-26T11:14:51,272 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dd0f2ef3a09410394398362f8984226, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732619685472 2024-11-26T11:14:51,272 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e09b6c7aeaa14353b91eac0bcd9e9a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732619685472 2024-11-26T11:14:51,273 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389ab5ed5abf4a55baef98b04110cd5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732619686722 2024-11-26T11:14:51,273 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 04f9039c07f14d6f95f5d2dfd1305187, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732619686722 2024-11-26T11:14:51,274 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a205c267de411c884a5aa6441abbab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732619687951 2024-11-26T11:14:51,274 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b902c88783466497b06d4df49e6a37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732619687951 2024-11-26T11:14:51,275 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0366b5b77c43c09088ad199203d823, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619689203 2024-11-26T11:14:51,275 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3685e7d62b4a6897b53983c3e84c4b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619689203 2024-11-26T11:14:51,333 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:51,333 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:51,333 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8affdbca5eb34ff4aa5a7811eff6c71f is 50, key is test_row_0/A:col10/1732619690358/Put/seqid=0 2024-11-26T11:14:51,334 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3eba2e3844f24cc9960e97fb8c6f506b is 50, key is test_row_0/B:col10/1732619690358/Put/seqid=0 2024-11-26T11:14:51,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741861_1037 (size=12275) 2024-11-26T11:14:51,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741862_1038 (size=12275) 2024-11-26T11:14:51,359 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8affdbca5eb34ff4aa5a7811eff6c71f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8affdbca5eb34ff4aa5a7811eff6c71f 2024-11-26T11:14:51,360 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3eba2e3844f24cc9960e97fb8c6f506b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3eba2e3844f24cc9960e97fb8c6f506b 2024-11-26T11:14:51,376 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into 8affdbca5eb34ff4aa5a7811eff6c71f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:51,377 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:51,377 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=12, startTime=1732619691267; duration=0sec 2024-11-26T11:14:51,377 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:51,377 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:14:51,377 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:51,379 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into 3eba2e3844f24cc9960e97fb8c6f506b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:51,379 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:51,379 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=12, startTime=1732619691268; duration=0sec 2024-11-26T11:14:51,379 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:51,379 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:14:51,381 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:51,381 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:14:51,381 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:51,381 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/933c0d1c4625448eb7d5c2bc4738e71f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/523f1408a496401e993e4c7afeb3d1d0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d5c284724607479b9fc4a3976caa93e5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/4e9ba4da81264ea1a414ea761d20c492] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=47.0 K 2024-11-26T11:14:51,382 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933c0d1c4625448eb7d5c2bc4738e71f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732619685472 2024-11-26T11:14:51,382 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523f1408a496401e993e4c7afeb3d1d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732619686722 2024-11-26T11:14:51,383 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5c284724607479b9fc4a3976caa93e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732619687951 2024-11-26T11:14:51,384 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9ba4da81264ea1a414ea761d20c492, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619689203 2024-11-26T11:14:51,412 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:51,413 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/83819a0e25fb4b6fb46e99a42c4e34cd is 50, key is test_row_0/C:col10/1732619690358/Put/seqid=0 2024-11-26T11:14:51,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741863_1039 (size=12275) 2024-11-26T11:14:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:51,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T11:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:51,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:51,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/2911728143dc4d0cafe0a5b44f841d17 is 50, key is test_row_0/A:col10/1732619691555/Put/seqid=0 2024-11-26T11:14:51,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741864_1040 (size=16931) 2024-11-26T11:14:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619751648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619751648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619751763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619751763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,837 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/83819a0e25fb4b6fb46e99a42c4e34cd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/83819a0e25fb4b6fb46e99a42c4e34cd 2024-11-26T11:14:51,845 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 83819a0e25fb4b6fb46e99a42c4e34cd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:51,845 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:51,845 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=12, startTime=1732619691268; duration=0sec 2024-11-26T11:14:51,845 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:51,845 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:14:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619751973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619751973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/2911728143dc4d0cafe0a5b44f841d17 2024-11-26T11:14:52,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/2ec36bad84e1409a8b01d16914fd21dc is 50, key is test_row_0/B:col10/1732619691555/Put/seqid=0 2024-11-26T11:14:52,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741865_1041 (size=12151) 2024-11-26T11:14:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T11:14:52,103 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:14:52,103 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T11:14:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T11:14:52,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:14:52,107 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:14:52,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:14:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T11:14:52,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T11:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:52,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619752283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619752283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619752302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,305 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619752313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619752313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,314 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8335 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,314 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in 77c23a8be60706f6017524015b3171ad of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8343 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T11:14:52,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:52,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T11:14:52,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/2ec36bad84e1409a8b01d16914fd21dc 2024-11-26T11:14:52,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/e16dab49fa0b48a48d085012881d2191 is 50, key is test_row_0/C:col10/1732619691555/Put/seqid=0 2024-11-26T11:14:52,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741866_1042 (size=12151) 2024-11-26T11:14:52,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T11:14:52,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T11:14:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T11:14:52,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619752793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619752793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:52,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/e16dab49fa0b48a48d085012881d2191 2024-11-26T11:14:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/2911728143dc4d0cafe0a5b44f841d17 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/2911728143dc4d0cafe0a5b44f841d17 2024-11-26T11:14:52,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/2911728143dc4d0cafe0a5b44f841d17, entries=250, sequenceid=145, filesize=16.5 K 2024-11-26T11:14:52,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/2ec36bad84e1409a8b01d16914fd21dc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/2ec36bad84e1409a8b01d16914fd21dc 2024-11-26T11:14:52,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/2ec36bad84e1409a8b01d16914fd21dc, entries=150, sequenceid=145, filesize=11.9 K 2024-11-26T11:14:52,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T11:14:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/e16dab49fa0b48a48d085012881d2191 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/e16dab49fa0b48a48d085012881d2191 2024-11-26T11:14:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:52,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:52,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/e16dab49fa0b48a48d085012881d2191, entries=150, sequenceid=145, filesize=11.9 K 2024-11-26T11:14:52,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 77c23a8be60706f6017524015b3171ad in 1335ms, sequenceid=145, compaction requested=false 2024-11-26T11:14:52,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:53,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T11:14:53,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:53,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:14:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:53,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:53,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/691ee9ecb52c4fd299c3fcd33c3307e1 is 50, key is test_row_0/A:col10/1732619691625/Put/seqid=0 2024-11-26T11:14:53,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741867_1043 (size=12151) 2024-11-26T11:14:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T11:14:53,446 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/691ee9ecb52c4fd299c3fcd33c3307e1 2024-11-26T11:14:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/81b03f5c46e44b3a837729d054900fbe is 50, key is test_row_0/B:col10/1732619691625/Put/seqid=0 2024-11-26T11:14:53,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741868_1044 (size=12151) 2024-11-26T11:14:53,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/81b03f5c46e44b3a837729d054900fbe 2024-11-26T11:14:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/54034317b2ed4789ba4e7af4851372c9 is 50, key is test_row_0/C:col10/1732619691625/Put/seqid=0 2024-11-26T11:14:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741869_1045 (size=12151) 2024-11-26T11:14:53,514 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/54034317b2ed4789ba4e7af4851372c9 2024-11-26T11:14:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/691ee9ecb52c4fd299c3fcd33c3307e1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/691ee9ecb52c4fd299c3fcd33c3307e1 2024-11-26T11:14:53,533 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/691ee9ecb52c4fd299c3fcd33c3307e1, entries=150, sequenceid=162, filesize=11.9 K 2024-11-26T11:14:53,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/81b03f5c46e44b3a837729d054900fbe as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81b03f5c46e44b3a837729d054900fbe 2024-11-26T11:14:53,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81b03f5c46e44b3a837729d054900fbe, entries=150, sequenceid=162, filesize=11.9 K 2024-11-26T11:14:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/54034317b2ed4789ba4e7af4851372c9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/54034317b2ed4789ba4e7af4851372c9 2024-11-26T11:14:53,557 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/54034317b2ed4789ba4e7af4851372c9, entries=150, sequenceid=162, filesize=11.9 K 2024-11-26T11:14:53,558 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 77c23a8be60706f6017524015b3171ad in 526ms, sequenceid=162, compaction requested=true 2024-11-26T11:14:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T11:14:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T11:14:53,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T11:14:53,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-26T11:14:53,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4630 sec 2024-11-26T11:14:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:53,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:14:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:53,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/c6f55b250d8246048f5bbe5a040eb23a is 50, key is test_row_0/A:col10/1732619693815/Put/seqid=0 2024-11-26T11:14:53,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741870_1046 (size=14541) 2024-11-26T11:14:53,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/c6f55b250d8246048f5bbe5a040eb23a 2024-11-26T11:14:53,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/339f4e7b01934f9ebc606e0ea781e14a is 50, key is test_row_0/B:col10/1732619693815/Put/seqid=0 2024-11-26T11:14:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741871_1047 (size=12151) 2024-11-26T11:14:53,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/339f4e7b01934f9ebc606e0ea781e14a 2024-11-26T11:14:53,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619693900, max timestamp=1732619693915Num uniques 859; 2024-11-26T11:14:53,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:53,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619693900, max timestamp=1732619693915Num uniques 859; 2024-11-26T11:14:53,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:53,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/1203be2bb3694fe0a12a7eee38b205e5 is 50, key is test_row_0/C:col10/1732619693815/Put/seqid=0 2024-11-26T11:14:53,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619693900, max timestamp=1732619693922Num uniques 859; 2024-11-26T11:14:53,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:53,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741872_1048 (size=12151) 2024-11-26T11:14:53,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/1203be2bb3694fe0a12a7eee38b205e5 2024-11-26T11:14:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/c6f55b250d8246048f5bbe5a040eb23a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c6f55b250d8246048f5bbe5a040eb23a 2024-11-26T11:14:53,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c6f55b250d8246048f5bbe5a040eb23a, entries=200, sequenceid=172, filesize=14.2 K 2024-11-26T11:14:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/339f4e7b01934f9ebc606e0ea781e14a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/339f4e7b01934f9ebc606e0ea781e14a 2024-11-26T11:14:54,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/339f4e7b01934f9ebc606e0ea781e14a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T11:14:54,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/1203be2bb3694fe0a12a7eee38b205e5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/1203be2bb3694fe0a12a7eee38b205e5 2024-11-26T11:14:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619754006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619754008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/1203be2bb3694fe0a12a7eee38b205e5, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T11:14:54,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 77c23a8be60706f6017524015b3171ad in 202ms, sequenceid=172, compaction requested=true 2024-11-26T11:14:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:14:54,026 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:14:54,027 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:14:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:54,029 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:54,029 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:14:54,029 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,029 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8affdbca5eb34ff4aa5a7811eff6c71f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/2911728143dc4d0cafe0a5b44f841d17, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/691ee9ecb52c4fd299c3fcd33c3307e1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c6f55b250d8246048f5bbe5a040eb23a] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=54.6 K 2024-11-26T11:14:54,029 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:54,029 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:14:54,029 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,030 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3eba2e3844f24cc9960e97fb8c6f506b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/2ec36bad84e1409a8b01d16914fd21dc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81b03f5c46e44b3a837729d054900fbe, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/339f4e7b01934f9ebc606e0ea781e14a] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=47.6 K 2024-11-26T11:14:54,030 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8affdbca5eb34ff4aa5a7811eff6c71f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619689203 2024-11-26T11:14:54,030 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eba2e3844f24cc9960e97fb8c6f506b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619689203 2024-11-26T11:14:54,031 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2911728143dc4d0cafe0a5b44f841d17, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732619690373 2024-11-26T11:14:54,031 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec36bad84e1409a8b01d16914fd21dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732619690373 2024-11-26T11:14:54,031 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b03f5c46e44b3a837729d054900fbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619691597 2024-11-26T11:14:54,031 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 691ee9ecb52c4fd299c3fcd33c3307e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619691597 2024-11-26T11:14:54,032 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6f55b250d8246048f5bbe5a040eb23a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732619693815 2024-11-26T11:14:54,032 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 339f4e7b01934f9ebc606e0ea781e14a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732619693815 2024-11-26T11:14:54,051 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:54,051 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/81124f9d658148ddb27bfa7adcf677df is 50, key is test_row_0/B:col10/1732619693815/Put/seqid=0 2024-11-26T11:14:54,060 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#37 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:54,061 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/d7fbcaacb4aa4f8dae3b6a1693297469 is 50, key is test_row_0/A:col10/1732619693815/Put/seqid=0 2024-11-26T11:14:54,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741874_1050 (size=12561) 2024-11-26T11:14:54,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741873_1049 (size=12561) 2024-11-26T11:14:54,093 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/d7fbcaacb4aa4f8dae3b6a1693297469 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d7fbcaacb4aa4f8dae3b6a1693297469 2024-11-26T11:14:54,110 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into d7fbcaacb4aa4f8dae3b6a1693297469(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:54,110 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:54,110 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=12, startTime=1732619694026; duration=0sec 2024-11-26T11:14:54,110 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:54,110 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:14:54,110 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:54,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:54,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:14:54,114 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,115 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/83819a0e25fb4b6fb46e99a42c4e34cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/e16dab49fa0b48a48d085012881d2191, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/54034317b2ed4789ba4e7af4851372c9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/1203be2bb3694fe0a12a7eee38b205e5] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=47.6 K 2024-11-26T11:14:54,116 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83819a0e25fb4b6fb46e99a42c4e34cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619689203 2024-11-26T11:14:54,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e16dab49fa0b48a48d085012881d2191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732619690373 2024-11-26T11:14:54,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54034317b2ed4789ba4e7af4851372c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619691597 2024-11-26T11:14:54,119 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1203be2bb3694fe0a12a7eee38b205e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732619693815 2024-11-26T11:14:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:54,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732619693977, max timestamp=1732619694128Num uniques 1018; 2024-11-26T11:14:54,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T11:14:54,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732619693977, max timestamp=1732619694128Num uniques 1018; 2024-11-26T11:14:54,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T11:14:54,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-26T11:14:54,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:54,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T11:14:54,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:54,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:54,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:54,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:54,145 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#38 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:54,146 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/9e0fed06ed534a80a300a5148ff66c71 is 50, key is test_row_0/C:col10/1732619693815/Put/seqid=0 2024-11-26T11:14:54,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/ae23a1fc22bb4929b3a0d6c4caf97e4c is 50, key is test_row_0/A:col10/1732619693990/Put/seqid=0 2024-11-26T11:14:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741875_1051 (size=12561) 2024-11-26T11:14:54,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741876_1052 (size=12151) 2024-11-26T11:14:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619754201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619754203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T11:14:54,253 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:14:54,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:14:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T11:14:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T11:14:54,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:14:54,258 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:14:54,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:14:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619754312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619754313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T11:14:54,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,493 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/81124f9d658148ddb27bfa7adcf677df as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81124f9d658148ddb27bfa7adcf677df 2024-11-26T11:14:54,503 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into 81124f9d658148ddb27bfa7adcf677df(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:54,503 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:54,503 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=12, startTime=1732619694026; duration=0sec 2024-11-26T11:14:54,503 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:54,503 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:14:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619754523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619754523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/ae23a1fc22bb4929b3a0d6c4caf97e4c 2024-11-26T11:14:54,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:54,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T11:14:54,578 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/9e0fed06ed534a80a300a5148ff66c71 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/9e0fed06ed534a80a300a5148ff66c71 2024-11-26T11:14:54,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/02b7148518294fb69ef28187e4d4a8d6 is 50, key is test_row_0/B:col10/1732619693990/Put/seqid=0 2024-11-26T11:14:54,588 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 9e0fed06ed534a80a300a5148ff66c71(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:54,588 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:54,588 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=12, startTime=1732619694027; duration=0sec 2024-11-26T11:14:54,588 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:54,588 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:14:54,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741877_1053 (size=12151) 2024-11-26T11:14:54,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619754832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619754833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:54,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:54,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T11:14:54,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/02b7148518294fb69ef28187e4d4a8d6 2024-11-26T11:14:55,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/10c4ad1ae3254b2ea332de85f5ff3efd is 50, key is test_row_0/C:col10/1732619693990/Put/seqid=0 2024-11-26T11:14:55,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741878_1054 (size=12151) 2024-11-26T11:14:55,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:55,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:55,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:55,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:55,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619755343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:55,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619755343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T11:14:55,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/10c4ad1ae3254b2ea332de85f5ff3efd 2024-11-26T11:14:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/ae23a1fc22bb4929b3a0d6c4caf97e4c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/ae23a1fc22bb4929b3a0d6c4caf97e4c 2024-11-26T11:14:55,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/ae23a1fc22bb4929b3a0d6c4caf97e4c, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T11:14:55,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/02b7148518294fb69ef28187e4d4a8d6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/02b7148518294fb69ef28187e4d4a8d6 2024-11-26T11:14:55,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/02b7148518294fb69ef28187e4d4a8d6, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T11:14:55,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/10c4ad1ae3254b2ea332de85f5ff3efd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c4ad1ae3254b2ea332de85f5ff3efd 2024-11-26T11:14:55,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c4ad1ae3254b2ea332de85f5ff3efd, entries=150, sequenceid=212, filesize=11.9 K 2024-11-26T11:14:55,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for 77c23a8be60706f6017524015b3171ad in 1314ms, sequenceid=212, compaction requested=false 2024-11-26T11:14:55,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:55,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T11:14:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,494 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:14:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/b37f68dc79ce4ae3a0b795643ca5a625 is 50, key is test_row_0/A:col10/1732619694190/Put/seqid=0 2024-11-26T11:14:55,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741879_1055 (size=12151) 2024-11-26T11:14:55,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/b37f68dc79ce4ae3a0b795643ca5a625 2024-11-26T11:14:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273 is 50, key is test_row_0/B:col10/1732619694190/Put/seqid=0 2024-11-26T11:14:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741880_1056 (size=12151) 2024-11-26T11:14:55,546 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273 2024-11-26T11:14:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/999933aac76e4d04bbba96d9a12181be is 50, key is test_row_0/C:col10/1732619694190/Put/seqid=0 2024-11-26T11:14:55,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741881_1057 (size=12151) 2024-11-26T11:14:55,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/999933aac76e4d04bbba96d9a12181be 2024-11-26T11:14:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/b37f68dc79ce4ae3a0b795643ca5a625 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b37f68dc79ce4ae3a0b795643ca5a625 2024-11-26T11:14:55,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b37f68dc79ce4ae3a0b795643ca5a625, entries=150, sequenceid=233, filesize=11.9 K 2024-11-26T11:14:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273 2024-11-26T11:14:55,598 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273, entries=150, sequenceid=233, filesize=11.9 K 2024-11-26T11:14:55,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/999933aac76e4d04bbba96d9a12181be as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/999933aac76e4d04bbba96d9a12181be 2024-11-26T11:14:55,611 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/999933aac76e4d04bbba96d9a12181be, entries=150, sequenceid=233, filesize=11.9 K 2024-11-26T11:14:55,613 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 77c23a8be60706f6017524015b3171ad in 119ms, sequenceid=233, compaction requested=true 2024-11-26T11:14:55,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:55,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:55,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T11:14:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T11:14:55,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T11:14:55,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3570 sec 2024-11-26T11:14:55,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3660 sec 2024-11-26T11:14:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T11:14:56,403 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:14:56,404 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:14:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T11:14:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T11:14:56,427 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:14:56,429 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:14:56,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:56,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:56,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/b421877b2983403ebfcfecb0aa677464 is 50, key is test_row_0/A:col10/1732619696434/Put/seqid=0 2024-11-26T11:14:56,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741882_1058 (size=16931) 2024-11-26T11:14:56,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/b421877b2983403ebfcfecb0aa677464 2024-11-26T11:14:56,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3a2c1160c98d41469f1a9bda732da5c7 is 50, key is test_row_0/B:col10/1732619696434/Put/seqid=0 2024-11-26T11:14:56,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741883_1059 (size=12151) 2024-11-26T11:14:56,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619696491, max timestamp=1732619696523Num uniques 859; 2024-11-26T11:14:56,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:56,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619696491, max timestamp=1732619696523Num uniques 859; 2024-11-26T11:14:56,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:14:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T11:14:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619756572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619756572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:56,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T11:14:56,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619756682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619756683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:56,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T11:14:56,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T11:14:56,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T11:14:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:56,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3a2c1160c98d41469f1a9bda732da5c7 2024-11-26T11:14:56,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619756903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:56,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:56,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619756904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:56,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/2ee95b04026644e4a446aea0461ef07d is 50, key is test_row_0/C:col10/1732619696434/Put/seqid=0 2024-11-26T11:14:56,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741884_1060 (size=12151) 2024-11-26T11:14:56,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/2ee95b04026644e4a446aea0461ef07d 2024-11-26T11:14:56,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/b421877b2983403ebfcfecb0aa677464 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b421877b2983403ebfcfecb0aa677464 2024-11-26T11:14:56,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b421877b2983403ebfcfecb0aa677464, entries=250, sequenceid=244, filesize=16.5 K 2024-11-26T11:14:56,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3a2c1160c98d41469f1a9bda732da5c7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3a2c1160c98d41469f1a9bda732da5c7 2024-11-26T11:14:56,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3a2c1160c98d41469f1a9bda732da5c7, entries=150, sequenceid=244, filesize=11.9 K 2024-11-26T11:14:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/2ee95b04026644e4a446aea0461ef07d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2ee95b04026644e4a446aea0461ef07d 2024-11-26T11:14:56,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2ee95b04026644e4a446aea0461ef07d, entries=150, sequenceid=244, filesize=11.9 K 2024-11-26T11:14:56,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 77c23a8be60706f6017524015b3171ad in 542ms, sequenceid=244, compaction requested=true 2024-11-26T11:14:56,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:56,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:14:56,993 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:56,993 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:56,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:14:56,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:56,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:14:56,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:56,994 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:56,995 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:14:56,995 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,995 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d7fbcaacb4aa4f8dae3b6a1693297469, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/ae23a1fc22bb4929b3a0d6c4caf97e4c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b37f68dc79ce4ae3a0b795643ca5a625, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b421877b2983403ebfcfecb0aa677464] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=52.5 K 2024-11-26T11:14:56,995 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:56,995 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:14:56,996 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:56,996 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81124f9d658148ddb27bfa7adcf677df, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/02b7148518294fb69ef28187e4d4a8d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3a2c1160c98d41469f1a9bda732da5c7] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=47.9 K 2024-11-26T11:14:56,996 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7fbcaacb4aa4f8dae3b6a1693297469, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732619693815 2024-11-26T11:14:56,997 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 81124f9d658148ddb27bfa7adcf677df, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732619693815 2024-11-26T11:14:56,997 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae23a1fc22bb4929b3a0d6c4caf97e4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732619693977 2024-11-26T11:14:56,998 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b7148518294fb69ef28187e4d4a8d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732619693977 2024-11-26T11:14:56,998 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b47b2b5f1b34e0eaa7e8c7f2d3b2273, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732619694190 2024-11-26T11:14:56,999 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a2c1160c98d41469f1a9bda732da5c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732619696428 2024-11-26T11:14:57,001 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37f68dc79ce4ae3a0b795643ca5a625, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732619694190 2024-11-26T11:14:57,002 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b421877b2983403ebfcfecb0aa677464, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732619696428 2024-11-26T11:14:57,020 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:57,021 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/61d5de79ec4f48448a30314b02b01856 is 50, key is test_row_0/B:col10/1732619696434/Put/seqid=0 2024-11-26T11:14:57,025 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:57,025 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/f63eb69171954e7394d1c5eaf55d1c88 is 50, key is test_row_0/A:col10/1732619696434/Put/seqid=0 2024-11-26T11:14:57,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741886_1062 (size=12697) 2024-11-26T11:14:57,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741885_1061 (size=12697) 2024-11-26T11:14:57,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T11:14:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:57,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-26T11:14:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:14:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T11:14:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/d2480a47eec54b138d9d8e2fd4f59193 is 50, key is test_row_0/A:col10/1732619696560/Put/seqid=0 2024-11-26T11:14:57,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741887_1063 (size=12301) 2024-11-26T11:14:57,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/d2480a47eec54b138d9d8e2fd4f59193 2024-11-26T11:14:57,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/dba9fdc00cce4903b6e9c3e80b378288 is 50, key is test_row_0/B:col10/1732619696560/Put/seqid=0 2024-11-26T11:14:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741888_1064 (size=12301) 2024-11-26T11:14:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:57,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619757237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619757238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619757342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619757353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,452 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/f63eb69171954e7394d1c5eaf55d1c88 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/f63eb69171954e7394d1c5eaf55d1c88 2024-11-26T11:14:57,457 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/61d5de79ec4f48448a30314b02b01856 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/61d5de79ec4f48448a30314b02b01856 2024-11-26T11:14:57,462 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into f63eb69171954e7394d1c5eaf55d1c88(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:57,462 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:57,462 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=12, startTime=1732619696992; duration=0sec 2024-11-26T11:14:57,462 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:57,462 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:14:57,463 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:14:57,467 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:14:57,467 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:14:57,467 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:57,467 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/9e0fed06ed534a80a300a5148ff66c71, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c4ad1ae3254b2ea332de85f5ff3efd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/999933aac76e4d04bbba96d9a12181be, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2ee95b04026644e4a446aea0461ef07d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=47.9 K 2024-11-26T11:14:57,467 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into 61d5de79ec4f48448a30314b02b01856(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:57,467 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:57,467 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=12, startTime=1732619696993; duration=0sec 2024-11-26T11:14:57,468 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:57,468 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:14:57,468 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0fed06ed534a80a300a5148ff66c71, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732619693815 2024-11-26T11:14:57,469 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c4ad1ae3254b2ea332de85f5ff3efd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732619693977 2024-11-26T11:14:57,469 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 999933aac76e4d04bbba96d9a12181be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732619694190 2024-11-26T11:14:57,470 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ee95b04026644e4a446aea0461ef07d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732619696428 2024-11-26T11:14:57,484 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:57,485 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/233780c7b03f4b4a8f631be270c73dcf is 50, key is test_row_0/C:col10/1732619696434/Put/seqid=0 2024-11-26T11:14:57,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741889_1065 (size=12697) 2024-11-26T11:14:57,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/dba9fdc00cce4903b6e9c3e80b378288 2024-11-26T11:14:57,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/f0309a4195a34a148ba351796813d5ed is 50, key is test_row_0/C:col10/1732619696560/Put/seqid=0 2024-11-26T11:14:57,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741890_1066 (size=12301) 2024-11-26T11:14:57,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619757552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T11:14:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619757563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619757862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:57,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619757872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:57,896 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/233780c7b03f4b4a8f631be270c73dcf as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/233780c7b03f4b4a8f631be270c73dcf 2024-11-26T11:14:57,904 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 233780c7b03f4b4a8f631be270c73dcf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:57,904 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:57,904 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=12, startTime=1732619696994; duration=0sec 2024-11-26T11:14:57,904 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:57,904 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:14:57,918 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/f0309a4195a34a148ba351796813d5ed 2024-11-26T11:14:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/d2480a47eec54b138d9d8e2fd4f59193 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d2480a47eec54b138d9d8e2fd4f59193 2024-11-26T11:14:57,929 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d2480a47eec54b138d9d8e2fd4f59193, entries=150, sequenceid=276, filesize=12.0 K 2024-11-26T11:14:57,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/dba9fdc00cce4903b6e9c3e80b378288 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/dba9fdc00cce4903b6e9c3e80b378288 2024-11-26T11:14:57,935 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/dba9fdc00cce4903b6e9c3e80b378288, entries=150, sequenceid=276, filesize=12.0 K 2024-11-26T11:14:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/f0309a4195a34a148ba351796813d5ed as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/f0309a4195a34a148ba351796813d5ed 2024-11-26T11:14:57,941 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/f0309a4195a34a148ba351796813d5ed, entries=150, sequenceid=276, filesize=12.0 K 2024-11-26T11:14:57,942 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=46.96 KB/48090 for 77c23a8be60706f6017524015b3171ad in 895ms, sequenceid=276, compaction requested=false 2024-11-26T11:14:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T11:14:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T11:14:57,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T11:14:57,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-11-26T11:14:57,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5430 sec 2024-11-26T11:14:58,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:14:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:58,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:14:58,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/355de4c0fe4d49a6a9172624b499d426 is 50, key is test_row_0/A:col10/1732619697236/Put/seqid=0 2024-11-26T11:14:58,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741891_1067 (size=12301) 2024-11-26T11:14:58,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619758443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619758447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:58,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619758563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619758563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T11:14:58,573 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:14:58,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:14:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T11:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:14:58,576 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:14:58,577 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:14:58,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:14:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:14:58,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:58,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:58,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:58,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:58,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:58,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:58,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619758773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:58,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619758773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:58,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/355de4c0fe4d49a6a9172624b499d426 2024-11-26T11:14:58,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/f047952991094aa9a54e93490ab6b318 is 50, key is test_row_0/B:col10/1732619697236/Put/seqid=0 2024-11-26T11:14:58,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741892_1068 (size=12301) 2024-11-26T11:14:58,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:58,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:58,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:58,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:58,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:58,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:14:59,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:59,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:59,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619759083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619759093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:59,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:14:59,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/f047952991094aa9a54e93490ab6b318 2024-11-26T11:14:59,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/3be8a3d02afc43f58611c117d96dfee8 is 50, key is test_row_0/C:col10/1732619697236/Put/seqid=0 2024-11-26T11:14:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741893_1069 (size=12301) 2024-11-26T11:14:59,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:59,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619759593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:59,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:14:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619759603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:14:59,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/3be8a3d02afc43f58611c117d96dfee8 2024-11-26T11:14:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/355de4c0fe4d49a6a9172624b499d426 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/355de4c0fe4d49a6a9172624b499d426 2024-11-26T11:14:59,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/355de4c0fe4d49a6a9172624b499d426, entries=150, sequenceid=291, filesize=12.0 K 2024-11-26T11:14:59,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/f047952991094aa9a54e93490ab6b318 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f047952991094aa9a54e93490ab6b318 2024-11-26T11:14:59,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:59,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:14:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:14:59,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f047952991094aa9a54e93490ab6b318, entries=150, sequenceid=291, filesize=12.0 K 2024-11-26T11:14:59,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/3be8a3d02afc43f58611c117d96dfee8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3be8a3d02afc43f58611c117d96dfee8 2024-11-26T11:14:59,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3be8a3d02afc43f58611c117d96dfee8, entries=150, sequenceid=291, filesize=12.0 K 2024-11-26T11:14:59,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 77c23a8be60706f6017524015b3171ad in 1287ms, sequenceid=291, compaction requested=true 2024-11-26T11:14:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:14:59,676 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:14:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:14:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:59,676 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:14:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:14:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:59,677 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:14:59,677 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:14:59,677 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:14:59,677 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:14:59,677 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,677 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,678 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/f63eb69171954e7394d1c5eaf55d1c88, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d2480a47eec54b138d9d8e2fd4f59193, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/355de4c0fe4d49a6a9172624b499d426] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.4 K 2024-11-26T11:14:59,678 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/61d5de79ec4f48448a30314b02b01856, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/dba9fdc00cce4903b6e9c3e80b378288, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f047952991094aa9a54e93490ab6b318] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.4 K 2024-11-26T11:14:59,679 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d5de79ec4f48448a30314b02b01856, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732619696428 2024-11-26T11:14:59,680 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f63eb69171954e7394d1c5eaf55d1c88, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732619696428 2024-11-26T11:14:59,680 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2480a47eec54b138d9d8e2fd4f59193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732619696558 2024-11-26T11:14:59,681 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting dba9fdc00cce4903b6e9c3e80b378288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732619696558 2024-11-26T11:14:59,682 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f047952991094aa9a54e93490ab6b318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732619697236 2024-11-26T11:14:59,682 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355de4c0fe4d49a6a9172624b499d426, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732619697236 2024-11-26T11:14:59,693 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:59,694 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:59,694 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/fb197c9eff1d4d06bac71d59299bca97 is 50, key is test_row_0/B:col10/1732619697236/Put/seqid=0 2024-11-26T11:14:59,694 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/dac61c04b8fe4e4f9ada2c0574d71aea is 50, key is test_row_0/A:col10/1732619697236/Put/seqid=0 2024-11-26T11:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:14:59,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741894_1070 (size=12949) 2024-11-26T11:14:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741895_1071 (size=12949) 2024-11-26T11:14:59,755 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/fb197c9eff1d4d06bac71d59299bca97 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/fb197c9eff1d4d06bac71d59299bca97 2024-11-26T11:14:59,775 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into fb197c9eff1d4d06bac71d59299bca97(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:59,775 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:59,775 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=13, startTime=1732619699676; duration=0sec 2024-11-26T11:14:59,775 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:14:59,775 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:14:59,776 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:14:59,777 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:14:59,777 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:14:59,778 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,778 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/233780c7b03f4b4a8f631be270c73dcf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/f0309a4195a34a148ba351796813d5ed, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3be8a3d02afc43f58611c117d96dfee8] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.4 K 2024-11-26T11:14:59,779 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 233780c7b03f4b4a8f631be270c73dcf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732619696428 2024-11-26T11:14:59,780 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f0309a4195a34a148ba351796813d5ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732619696558 2024-11-26T11:14:59,781 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3be8a3d02afc43f58611c117d96dfee8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732619697236 2024-11-26T11:14:59,806 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:14:59,807 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0762d3b3bfc7499396b3adcfde72690f is 50, key is test_row_0/C:col10/1732619697236/Put/seqid=0 2024-11-26T11:14:59,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T11:14:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:14:59,812 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:14:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:59,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:14:59,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:14:59,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741896_1072 (size=12949) 2024-11-26T11:14:59,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/c4f725ec42104c2dbe94aa4e0d73fb28 is 50, key is test_row_0/A:col10/1732619698437/Put/seqid=0 2024-11-26T11:14:59,851 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0762d3b3bfc7499396b3adcfde72690f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0762d3b3bfc7499396b3adcfde72690f 2024-11-26T11:14:59,865 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 0762d3b3bfc7499396b3adcfde72690f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:14:59,865 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:14:59,866 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=13, startTime=1732619699676; duration=0sec 2024-11-26T11:14:59,866 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:14:59,866 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:14:59,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741897_1073 (size=12301) 2024-11-26T11:15:00,127 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/dac61c04b8fe4e4f9ada2c0574d71aea as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/dac61c04b8fe4e4f9ada2c0574d71aea 2024-11-26T11:15:00,133 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into dac61c04b8fe4e4f9ada2c0574d71aea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:00,134 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:00,134 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=13, startTime=1732619699676; duration=0sec 2024-11-26T11:15:00,134 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:00,134 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:15:00,277 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/c4f725ec42104c2dbe94aa4e0d73fb28 2024-11-26T11:15:00,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e916e93d50944a8995abe7092194c0f0 is 50, key is test_row_0/B:col10/1732619698437/Put/seqid=0 2024-11-26T11:15:00,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741898_1074 (size=12301) 2024-11-26T11:15:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:00,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:15:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619760638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619760638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:00,699 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e916e93d50944a8995abe7092194c0f0 2024-11-26T11:15:00,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01 is 50, key is test_row_0/C:col10/1732619698437/Put/seqid=0 2024-11-26T11:15:00,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741899_1075 (size=12301) 2024-11-26T11:15:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:15:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619760753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619760753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:00,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619760962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:00,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619760963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,119 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01 2024-11-26T11:15:01,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/c4f725ec42104c2dbe94aa4e0d73fb28 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c4f725ec42104c2dbe94aa4e0d73fb28 2024-11-26T11:15:01,140 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c4f725ec42104c2dbe94aa4e0d73fb28, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T11:15:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e916e93d50944a8995abe7092194c0f0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e916e93d50944a8995abe7092194c0f0 2024-11-26T11:15:01,147 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e916e93d50944a8995abe7092194c0f0, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T11:15:01,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01 2024-11-26T11:15:01,157 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T11:15:01,159 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 77c23a8be60706f6017524015b3171ad in 1346ms, sequenceid=312, compaction requested=false 2024-11-26T11:15:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T11:15:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T11:15:01,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T11:15:01,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5840 sec 2024-11-26T11:15:01,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5920 sec 2024-11-26T11:15:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:01,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:15:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:01,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/31175a16f4d541d1a2c08d40c3e572b6 is 50, key is test_row_0/A:col10/1732619700631/Put/seqid=0 2024-11-26T11:15:01,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741900_1076 (size=14741) 2024-11-26T11:15:01,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/31175a16f4d541d1a2c08d40c3e572b6 2024-11-26T11:15:01,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45 is 50, key is test_row_0/B:col10/1732619700631/Put/seqid=0 2024-11-26T11:15:01,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741901_1077 (size=12301) 2024-11-26T11:15:01,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45 2024-11-26T11:15:01,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a2fa36c6469d436f99c7b3b9256b168e is 50, key is test_row_0/C:col10/1732619700631/Put/seqid=0 2024-11-26T11:15:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619761321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619761325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741902_1078 (size=12301) 2024-11-26T11:15:01,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a2fa36c6469d436f99c7b3b9256b168e 2024-11-26T11:15:01,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/31175a16f4d541d1a2c08d40c3e572b6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/31175a16f4d541d1a2c08d40c3e572b6 2024-11-26T11:15:01,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/31175a16f4d541d1a2c08d40c3e572b6, entries=200, sequenceid=328, filesize=14.4 K 2024-11-26T11:15:01,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45 2024-11-26T11:15:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T11:15:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a2fa36c6469d436f99c7b3b9256b168e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a2fa36c6469d436f99c7b3b9256b168e 2024-11-26T11:15:01,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a2fa36c6469d436f99c7b3b9256b168e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T11:15:01,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 77c23a8be60706f6017524015b3171ad in 134ms, sequenceid=328, compaction requested=true 2024-11-26T11:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:01,409 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:01,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:01,410 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:01,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:01,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:01,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:01,411 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:01,411 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:01,411 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:15:01,411 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:15:01,411 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:01,411 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:01,411 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/fb197c9eff1d4d06bac71d59299bca97, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e916e93d50944a8995abe7092194c0f0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.7 K 2024-11-26T11:15:01,411 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/dac61c04b8fe4e4f9ada2c0574d71aea, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c4f725ec42104c2dbe94aa4e0d73fb28, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/31175a16f4d541d1a2c08d40c3e572b6] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=39.1 K 2024-11-26T11:15:01,411 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting dac61c04b8fe4e4f9ada2c0574d71aea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732619697236 2024-11-26T11:15:01,411 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting fb197c9eff1d4d06bac71d59299bca97, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732619697236 2024-11-26T11:15:01,412 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4f725ec42104c2dbe94aa4e0d73fb28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619698437 2024-11-26T11:15:01,412 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e916e93d50944a8995abe7092194c0f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619698437 2024-11-26T11:15:01,412 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31175a16f4d541d1a2c08d40c3e572b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732619700622 2024-11-26T11:15:01,412 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0272aaa4d4fb4e5ba3e2be69b5ed0f45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732619700622 2024-11-26T11:15:01,422 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:01,422 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:01,422 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8f4eaed79a804d0eac7844cbd416ab69 is 50, key is test_row_0/A:col10/1732619700631/Put/seqid=0 2024-11-26T11:15:01,422 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/c006af517f064e69bbf0dcdccb377deb is 50, key is test_row_0/B:col10/1732619700631/Put/seqid=0 2024-11-26T11:15:01,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741904_1080 (size=13051) 2024-11-26T11:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:01,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:15:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:01,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:01,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741903_1079 (size=13051) 2024-11-26T11:15:01,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/52751f5eb9834a928e7bdaa4056161d6 is 50, key is test_row_0/A:col10/1732619701433/Put/seqid=0 2024-11-26T11:15:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741905_1081 (size=12301) 2024-11-26T11:15:01,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619761464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619761465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619761583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619761583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619761793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619761793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:01,843 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/8f4eaed79a804d0eac7844cbd416ab69 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8f4eaed79a804d0eac7844cbd416ab69 2024-11-26T11:15:01,848 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/c006af517f064e69bbf0dcdccb377deb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/c006af517f064e69bbf0dcdccb377deb 2024-11-26T11:15:01,854 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into 8f4eaed79a804d0eac7844cbd416ab69(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:01,854 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into c006af517f064e69bbf0dcdccb377deb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:01,854 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=13, startTime=1732619701409; duration=0sec 2024-11-26T11:15:01,854 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=13, startTime=1732619701409; duration=0sec 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:15:01,854 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:01,855 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:01,855 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:15:01,856 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:01,856 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0762d3b3bfc7499396b3adcfde72690f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a2fa36c6469d436f99c7b3b9256b168e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.7 K 2024-11-26T11:15:01,856 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0762d3b3bfc7499396b3adcfde72690f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732619697236 2024-11-26T11:15:01,857 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4a00fd0d9e4ed8a2989f4e08fcbd01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619698437 2024-11-26T11:15:01,858 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2fa36c6469d436f99c7b3b9256b168e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732619700622 2024-11-26T11:15:01,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/52751f5eb9834a928e7bdaa4056161d6 2024-11-26T11:15:01,886 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:01,887 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/10c3b85ab16c4a94b1f777c0b7f95301 is 50, key is test_row_0/C:col10/1732619700631/Put/seqid=0 2024-11-26T11:15:01,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/31cbb2e92785491ab5c35f88f5ee9612 is 50, key is test_row_0/B:col10/1732619701433/Put/seqid=0 2024-11-26T11:15:01,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741907_1083 (size=12301) 2024-11-26T11:15:01,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741906_1082 (size=13051) 2024-11-26T11:15:01,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/31cbb2e92785491ab5c35f88f5ee9612 2024-11-26T11:15:01,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0027364cd5d04f88812ef2d333194227 is 50, key is test_row_0/C:col10/1732619701433/Put/seqid=0 2024-11-26T11:15:01,924 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/10c3b85ab16c4a94b1f777c0b7f95301 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c3b85ab16c4a94b1f777c0b7f95301 2024-11-26T11:15:01,931 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 10c3b85ab16c4a94b1f777c0b7f95301(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:01,932 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:01,932 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=13, startTime=1732619701410; duration=0sec 2024-11-26T11:15:01,932 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:01,932 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:15:01,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741908_1084 (size=12301) 2024-11-26T11:15:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619762103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619762103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0027364cd5d04f88812ef2d333194227 2024-11-26T11:15:02,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/52751f5eb9834a928e7bdaa4056161d6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/52751f5eb9834a928e7bdaa4056161d6 2024-11-26T11:15:02,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/52751f5eb9834a928e7bdaa4056161d6, entries=150, sequenceid=347, filesize=12.0 K 2024-11-26T11:15:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/31cbb2e92785491ab5c35f88f5ee9612 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/31cbb2e92785491ab5c35f88f5ee9612 2024-11-26T11:15:02,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/31cbb2e92785491ab5c35f88f5ee9612, entries=150, sequenceid=347, filesize=12.0 K 2024-11-26T11:15:02,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/0027364cd5d04f88812ef2d333194227 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0027364cd5d04f88812ef2d333194227 2024-11-26T11:15:02,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0027364cd5d04f88812ef2d333194227, entries=150, sequenceid=347, filesize=12.0 K 2024-11-26T11:15:02,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 77c23a8be60706f6017524015b3171ad in 940ms, sequenceid=347, compaction requested=false 2024-11-26T11:15:02,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:02,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:15:02,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:02,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/e3cc4e729bbd4001afdef0bd6883b8c5 is 50, key is test_row_0/A:col10/1732619701463/Put/seqid=0 2024-11-26T11:15:02,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741909_1085 (size=17181) 2024-11-26T11:15:02,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/e3cc4e729bbd4001afdef0bd6883b8c5 2024-11-26T11:15:02,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b4805ee0df254cc18b95566177411ee7 is 50, key is test_row_0/B:col10/1732619701463/Put/seqid=0 2024-11-26T11:15:02,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741910_1086 (size=12301) 2024-11-26T11:15:02,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619762417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619762418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619762420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619762533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619762533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619762533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619762613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619762613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T11:15:02,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:02,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619762742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:02,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-26T11:15:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T11:15:02,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:02,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:02,748 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619762743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619762743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:02,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:02,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b4805ee0df254cc18b95566177411ee7 2024-11-26T11:15:02,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/6b75069222ab494dbe38739a291095df is 50, key is test_row_0/C:col10/1732619701463/Put/seqid=0 2024-11-26T11:15:02,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741911_1087 (size=12301) 2024-11-26T11:15:02,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/6b75069222ab494dbe38739a291095df 2024-11-26T11:15:02,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/e3cc4e729bbd4001afdef0bd6883b8c5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/e3cc4e729bbd4001afdef0bd6883b8c5 2024-11-26T11:15:02,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/e3cc4e729bbd4001afdef0bd6883b8c5, entries=250, sequenceid=366, filesize=16.8 K 2024-11-26T11:15:02,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b4805ee0df254cc18b95566177411ee7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b4805ee0df254cc18b95566177411ee7 2024-11-26T11:15:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T11:15:02,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b4805ee0df254cc18b95566177411ee7, entries=150, sequenceid=366, filesize=12.0 K 2024-11-26T11:15:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/6b75069222ab494dbe38739a291095df as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6b75069222ab494dbe38739a291095df 2024-11-26T11:15:02,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6b75069222ab494dbe38739a291095df, entries=150, sequenceid=366, filesize=12.0 K 2024-11-26T11:15:02,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 77c23a8be60706f6017524015b3171ad in 487ms, sequenceid=366, compaction requested=true 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:02,872 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:02,872 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:02,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:02,873 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:02,873 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:02,873 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:15:02,873 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:15:02,873 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:02,873 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:02,874 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8f4eaed79a804d0eac7844cbd416ab69, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/52751f5eb9834a928e7bdaa4056161d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/e3cc4e729bbd4001afdef0bd6883b8c5] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=41.5 K 2024-11-26T11:15:02,874 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/c006af517f064e69bbf0dcdccb377deb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/31cbb2e92785491ab5c35f88f5ee9612, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b4805ee0df254cc18b95566177411ee7] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.8 K 2024-11-26T11:15:02,874 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c006af517f064e69bbf0dcdccb377deb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732619700622 2024-11-26T11:15:02,874 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4eaed79a804d0eac7844cbd416ab69, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732619700622 2024-11-26T11:15:02,875 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52751f5eb9834a928e7bdaa4056161d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732619701311 2024-11-26T11:15:02,875 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cbb2e92785491ab5c35f88f5ee9612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732619701311 2024-11-26T11:15:02,875 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3cc4e729bbd4001afdef0bd6883b8c5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732619701458 2024-11-26T11:15:02,875 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b4805ee0df254cc18b95566177411ee7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732619701463 2024-11-26T11:15:02,892 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:02,893 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/1f9f6163659f44f5b898173ff17ce57e is 50, key is test_row_0/A:col10/1732619701463/Put/seqid=0 2024-11-26T11:15:02,894 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#75 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:02,895 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e88fe70a85db48fe9177ab3020386882 is 50, key is test_row_0/B:col10/1732619701463/Put/seqid=0 2024-11-26T11:15:02,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T11:15:02,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:02,901 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741912_1088 (size=13153) 2024-11-26T11:15:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/844a3f65d0f84890a24fcd8343b53f42 is 50, key is test_row_0/A:col10/1732619702416/Put/seqid=0 2024-11-26T11:15:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741913_1089 (size=13153) 2024-11-26T11:15:02,923 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/e88fe70a85db48fe9177ab3020386882 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e88fe70a85db48fe9177ab3020386882 2024-11-26T11:15:02,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741914_1090 (size=12301) 2024-11-26T11:15:02,926 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/844a3f65d0f84890a24fcd8343b53f42 2024-11-26T11:15:02,932 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into e88fe70a85db48fe9177ab3020386882(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:02,932 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:02,932 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=13, startTime=1732619702872; duration=0sec 2024-11-26T11:15:02,932 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:02,933 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:15:02,933 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:02,934 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:02,934 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:15:02,934 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:02,934 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c3b85ab16c4a94b1f777c0b7f95301, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0027364cd5d04f88812ef2d333194227, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6b75069222ab494dbe38739a291095df] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.8 K 2024-11-26T11:15:02,934 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c3b85ab16c4a94b1f777c0b7f95301, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732619700622 2024-11-26T11:15:02,935 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0027364cd5d04f88812ef2d333194227, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732619701311 2024-11-26T11:15:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/f3483b9a314a425f9e6cb9b5aa0543cd is 50, key is test_row_0/B:col10/1732619702416/Put/seqid=0 2024-11-26T11:15:02,936 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b75069222ab494dbe38739a291095df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732619701463 2024-11-26T11:15:02,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741915_1091 (size=12301) 2024-11-26T11:15:02,950 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:02,952 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/f3483b9a314a425f9e6cb9b5aa0543cd 2024-11-26T11:15:02,953 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/90d2b79c97ca4c8380de5c6ad0e4246b is 50, key is test_row_0/C:col10/1732619701463/Put/seqid=0 2024-11-26T11:15:02,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741916_1092 (size=13153) 2024-11-26T11:15:02,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/2308f2a8b7c74af3a7974888f6b18644 is 50, key is test_row_0/C:col10/1732619702416/Put/seqid=0 2024-11-26T11:15:02,973 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/90d2b79c97ca4c8380de5c6ad0e4246b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/90d2b79c97ca4c8380de5c6ad0e4246b 2024-11-26T11:15:02,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741917_1093 (size=12301) 2024-11-26T11:15:02,989 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into 90d2b79c97ca4c8380de5c6ad0e4246b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:02,989 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:02,989 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=13, startTime=1732619702872; duration=0sec 2024-11-26T11:15:02,989 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:02,989 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:15:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:03,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. as already flushing 2024-11-26T11:15:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T11:15:03,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619763073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619763074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619763076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619763182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619763182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619763182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,314 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/1f9f6163659f44f5b898173ff17ce57e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/1f9f6163659f44f5b898173ff17ce57e 2024-11-26T11:15:03,320 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into 1f9f6163659f44f5b898173ff17ce57e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:03,320 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:03,320 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=13, startTime=1732619702872; duration=0sec 2024-11-26T11:15:03,320 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:03,320 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:15:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T11:15:03,384 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/2308f2a8b7c74af3a7974888f6b18644 2024-11-26T11:15:03,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/844a3f65d0f84890a24fcd8343b53f42 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/844a3f65d0f84890a24fcd8343b53f42 2024-11-26T11:15:03,395 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/844a3f65d0f84890a24fcd8343b53f42, entries=150, sequenceid=382, filesize=12.0 K 2024-11-26T11:15:03,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/f3483b9a314a425f9e6cb9b5aa0543cd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f3483b9a314a425f9e6cb9b5aa0543cd 2024-11-26T11:15:03,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619763392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f3483b9a314a425f9e6cb9b5aa0543cd, entries=150, sequenceid=382, filesize=12.0 K 2024-11-26T11:15:03,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/2308f2a8b7c74af3a7974888f6b18644 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2308f2a8b7c74af3a7974888f6b18644 2024-11-26T11:15:03,412 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2308f2a8b7c74af3a7974888f6b18644, entries=150, sequenceid=382, filesize=12.0 K 2024-11-26T11:15:03,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 77c23a8be60706f6017524015b3171ad in 512ms, sequenceid=382, compaction requested=false 2024-11-26T11:15:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-26T11:15:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-26T11:15:03,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T11:15:03,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 667 msec 2024-11-26T11:15:03,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 674 msec 2024-11-26T11:15:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:03,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T11:15:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:03,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/aa679491ebff4827b81299015d0bdbe8 is 50, key is test_row_0/A:col10/1732619703063/Put/seqid=0 2024-11-26T11:15:03,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741918_1094 (size=17181) 2024-11-26T11:15:03,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619763666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619763673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619763703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619763703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619763703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,752 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T11:15:03,752 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:03,754 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T11:15:03,754 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:03,757 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T11:15:03,757 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:03,757 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,757 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,757 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,757 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,757 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:03,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:03,757 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:03,758 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T11:15:03,758 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:03,758 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,758 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:03,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:03,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619763783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619763783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T11:15:03,884 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:03,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619763993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619763994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/aa679491ebff4827b81299015d0bdbe8 2024-11-26T11:15:04,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b0569277438c41a59411b945919ab7da is 50, key is test_row_0/B:col10/1732619703063/Put/seqid=0 2024-11-26T11:15:04,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741919_1095 (size=12301) 2024-11-26T11:15:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33158 deadline: 1732619764214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33142 deadline: 1732619764214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33114 deadline: 1732619764215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,262 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T11:15:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619764303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619764304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b0569277438c41a59411b945919ab7da 2024-11-26T11:15:04,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/6d70bf6fed98440a86276edd87b08e35 is 50, key is test_row_0/C:col10/1732619703063/Put/seqid=0 2024-11-26T11:15:04,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741920_1096 (size=12301) 2024-11-26T11:15:04,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33122 deadline: 1732619764813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33172 deadline: 1732619764813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:04,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/6d70bf6fed98440a86276edd87b08e35 2024-11-26T11:15:04,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/aa679491ebff4827b81299015d0bdbe8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/aa679491ebff4827b81299015d0bdbe8 2024-11-26T11:15:04,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/aa679491ebff4827b81299015d0bdbe8, entries=250, sequenceid=403, filesize=16.8 K 2024-11-26T11:15:04,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b0569277438c41a59411b945919ab7da as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b0569277438c41a59411b945919ab7da 2024-11-26T11:15:04,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b0569277438c41a59411b945919ab7da, entries=150, sequenceid=403, filesize=12.0 K 2024-11-26T11:15:04,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/6d70bf6fed98440a86276edd87b08e35 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6d70bf6fed98440a86276edd87b08e35 2024-11-26T11:15:04,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6d70bf6fed98440a86276edd87b08e35, entries=150, sequenceid=403, filesize=12.0 K 2024-11-26T11:15:04,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 77c23a8be60706f6017524015b3171ad in 1283ms, sequenceid=403, compaction requested=true 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:04,919 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:04,919 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 77c23a8be60706f6017524015b3171ad:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:04,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:04,920 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:04,920 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:04,920 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/A is initiating minor compaction (all files) 2024-11-26T11:15:04,920 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/B is initiating minor compaction (all files) 2024-11-26T11:15:04,920 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/B in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:04,920 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/A in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:04,920 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/1f9f6163659f44f5b898173ff17ce57e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/844a3f65d0f84890a24fcd8343b53f42, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/aa679491ebff4827b81299015d0bdbe8] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=41.6 K 2024-11-26T11:15:04,920 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e88fe70a85db48fe9177ab3020386882, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f3483b9a314a425f9e6cb9b5aa0543cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b0569277438c41a59411b945919ab7da] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.9 K 2024-11-26T11:15:04,921 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9f6163659f44f5b898173ff17ce57e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732619701463 2024-11-26T11:15:04,921 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e88fe70a85db48fe9177ab3020386882, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732619701463 2024-11-26T11:15:04,921 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f3483b9a314a425f9e6cb9b5aa0543cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732619702416 2024-11-26T11:15:04,921 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844a3f65d0f84890a24fcd8343b53f42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732619702416 2024-11-26T11:15:04,921 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa679491ebff4827b81299015d0bdbe8, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732619703060 2024-11-26T11:15:04,921 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b0569277438c41a59411b945919ab7da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732619703060 2024-11-26T11:15:04,930 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:04,930 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/695d75f1d3894c1d8670009f6348b528 is 50, key is test_row_0/B:col10/1732619703063/Put/seqid=0 2024-11-26T11:15:04,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741921_1097 (size=13255) 2024-11-26T11:15:04,956 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#A#compaction#85 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:04,957 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/28479b66f9e7473fbf1b1214c81e1193 is 50, key is test_row_0/A:col10/1732619703063/Put/seqid=0 2024-11-26T11:15:04,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741922_1098 (size=13255) 2024-11-26T11:15:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:05,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:15:05,226 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T11:15:05,226 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T11:15:05,226 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T11:15:05,226 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:05,226 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:05,226 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:05,226 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,226 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,226 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:05,226 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,226 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:05,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:05,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:05,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:05,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:05,226 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:05,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:05,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:05,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/5ea2cb67c4264272b745d562927e7a84 is 50, key is test_row_0/A:col10/1732619705223/Put/seqid=0 2024-11-26T11:15:05,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741923_1099 (size=12301) 2024-11-26T11:15:05,344 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/695d75f1d3894c1d8670009f6348b528 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/695d75f1d3894c1d8670009f6348b528 2024-11-26T11:15:05,351 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/B of 77c23a8be60706f6017524015b3171ad into 695d75f1d3894c1d8670009f6348b528(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:05,351 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:05,351 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/B, priority=13, startTime=1732619704919; duration=0sec 2024-11-26T11:15:05,351 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:05,351 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:B 2024-11-26T11:15:05,351 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:05,353 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:05,353 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 77c23a8be60706f6017524015b3171ad/C is initiating minor compaction (all files) 2024-11-26T11:15:05,353 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 77c23a8be60706f6017524015b3171ad/C in TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:05,353 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/90d2b79c97ca4c8380de5c6ad0e4246b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2308f2a8b7c74af3a7974888f6b18644, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6d70bf6fed98440a86276edd87b08e35] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp, totalSize=36.9 K 2024-11-26T11:15:05,354 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d2b79c97ca4c8380de5c6ad0e4246b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732619701463 2024-11-26T11:15:05,354 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 2308f2a8b7c74af3a7974888f6b18644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732619702416 2024-11-26T11:15:05,354 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d70bf6fed98440a86276edd87b08e35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732619703060 2024-11-26T11:15:05,365 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 77c23a8be60706f6017524015b3171ad#C#compaction#87 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:05,366 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a8f7136d672946ed8fd4e71fb4d54868 is 50, key is test_row_0/C:col10/1732619703063/Put/seqid=0 2024-11-26T11:15:05,368 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/28479b66f9e7473fbf1b1214c81e1193 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/28479b66f9e7473fbf1b1214c81e1193 2024-11-26T11:15:05,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741924_1100 (size=13255) 2024-11-26T11:15:05,374 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/A of 77c23a8be60706f6017524015b3171ad into 28479b66f9e7473fbf1b1214c81e1193(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:05,374 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:05,375 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/A, priority=13, startTime=1732619704919; duration=0sec 2024-11-26T11:15:05,375 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:05,375 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:A 2024-11-26T11:15:05,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/5ea2cb67c4264272b745d562927e7a84 2024-11-26T11:15:05,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3c764192eb4a4dff93d7c55d3a609c4d is 50, key is test_row_0/B:col10/1732619705223/Put/seqid=0 2024-11-26T11:15:05,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741925_1101 (size=12301) 2024-11-26T11:15:05,786 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/a8f7136d672946ed8fd4e71fb4d54868 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a8f7136d672946ed8fd4e71fb4d54868 2024-11-26T11:15:05,792 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 77c23a8be60706f6017524015b3171ad/C of 77c23a8be60706f6017524015b3171ad into a8f7136d672946ed8fd4e71fb4d54868(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:05,792 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:05,792 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., storeName=77c23a8be60706f6017524015b3171ad/C, priority=13, startTime=1732619704919; duration=0sec 2024-11-26T11:15:05,793 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:05,793 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 77c23a8be60706f6017524015b3171ad:C 2024-11-26T11:15:05,825 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T11:15:05,825 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:05,825 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,825 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,825 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T11:15:05,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:05,825 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:05,826 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,826 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:05,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T11:15:05,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-11-26T11:15:05,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-26T11:15:05,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-26T11:15:05,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3067 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2988 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1319 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3950 rows 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1329 2024-11-26T11:15:05,827 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3980 rows 2024-11-26T11:15:05,827 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:15:05,827 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:15:05,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:05,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:05,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T11:15:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T11:15:05,842 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619705841"}]},"ts":"1732619705841"} 2024-11-26T11:15:05,844 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T11:15:05,844 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T11:15:05,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:15:05,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, UNASSIGN}] 2024-11-26T11:15:05,851 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, UNASSIGN 2024-11-26T11:15:05,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=77c23a8be60706f6017524015b3171ad, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:05,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, UNASSIGN because future has completed 2024-11-26T11:15:05,856 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:15:05,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:15:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T11:15:06,012 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:06,012 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:15:06,013 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 77c23a8be60706f6017524015b3171ad, disabling compactions & flushes 2024-11-26T11:15:06,013 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:06,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3c764192eb4a4dff93d7c55d3a609c4d 2024-11-26T11:15:06,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc is 50, key is test_row_0/C:col10/1732619705223/Put/seqid=0 2024-11-26T11:15:06,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741926_1102 (size=12301) 2024-11-26T11:15:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T11:15:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T11:15:06,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc 2024-11-26T11:15:06,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/5ea2cb67c4264272b745d562927e7a84 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/5ea2cb67c4264272b745d562927e7a84 2024-11-26T11:15:06,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/5ea2cb67c4264272b745d562927e7a84, entries=150, sequenceid=420, filesize=12.0 K 2024-11-26T11:15:06,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/3c764192eb4a4dff93d7c55d3a609c4d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3c764192eb4a4dff93d7c55d3a609c4d 2024-11-26T11:15:06,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3c764192eb4a4dff93d7c55d3a609c4d, entries=150, sequenceid=420, filesize=12.0 K 2024-11-26T11:15:06,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc 2024-11-26T11:15:06,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc, entries=150, sequenceid=420, filesize=12.0 K 2024-11-26T11:15:06,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for 77c23a8be60706f6017524015b3171ad in 1344ms, sequenceid=420, compaction requested=false 2024-11-26T11:15:06,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 77c23a8be60706f6017524015b3171ad: 2024-11-26T11:15:06,569 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:06,569 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:06,569 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. after waiting 0 ms 2024-11-26T11:15:06,569 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:06,569 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 77c23a8be60706f6017524015b3171ad 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-26T11:15:06,569 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=A 2024-11-26T11:15:06,570 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:06,570 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=B 2024-11-26T11:15:06,570 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:06,570 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 77c23a8be60706f6017524015b3171ad, store=C 2024-11-26T11:15:06,570 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:06,573 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/94557dbf9a7c46cf8268173e7341ec9d is 50, key is test_row_0/A:col10/1732619705823/Put/seqid=0 2024-11-26T11:15:06,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741927_1103 (size=9857) 2024-11-26T11:15:06,978 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/94557dbf9a7c46cf8268173e7341ec9d 2024-11-26T11:15:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T11:15:06,987 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b15f84e3676d4b48bf4c0a91465269b3 is 50, key is test_row_0/B:col10/1732619705823/Put/seqid=0 2024-11-26T11:15:06,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741928_1104 (size=9857) 2024-11-26T11:15:06,993 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b15f84e3676d4b48bf4c0a91465269b3 2024-11-26T11:15:07,006 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/5cf80f9fcaed457e8830020fdec6face is 50, key is test_row_0/C:col10/1732619705823/Put/seqid=0 2024-11-26T11:15:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741929_1105 (size=9857) 2024-11-26T11:15:07,411 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/5cf80f9fcaed457e8830020fdec6face 2024-11-26T11:15:07,417 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/A/94557dbf9a7c46cf8268173e7341ec9d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/94557dbf9a7c46cf8268173e7341ec9d 2024-11-26T11:15:07,422 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/94557dbf9a7c46cf8268173e7341ec9d, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T11:15:07,423 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/B/b15f84e3676d4b48bf4c0a91465269b3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b15f84e3676d4b48bf4c0a91465269b3 2024-11-26T11:15:07,428 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b15f84e3676d4b48bf4c0a91465269b3, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T11:15:07,429 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/.tmp/C/5cf80f9fcaed457e8830020fdec6face as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/5cf80f9fcaed457e8830020fdec6face 2024-11-26T11:15:07,434 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/5cf80f9fcaed457e8830020fdec6face, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T11:15:07,435 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 77c23a8be60706f6017524015b3171ad in 866ms, sequenceid=428, compaction requested=true 2024-11-26T11:15:07,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8477a500078b4d3faa051bb78388210a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7b88cc2bf5d94c9088525f50ad7fc96c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/48fa92fb55ae42918a1db6fb221b1a74, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/df31f2af58fc4a8fa29b1a7976a3d83a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7dd0f2ef3a09410394398362f8984226, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/389ab5ed5abf4a55baef98b04110cd5a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/44a205c267de411c884a5aa6441abbab, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/3e3685e7d62b4a6897b53983c3e84c4b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8affdbca5eb34ff4aa5a7811eff6c71f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/2911728143dc4d0cafe0a5b44f841d17, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/691ee9ecb52c4fd299c3fcd33c3307e1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c6f55b250d8246048f5bbe5a040eb23a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d7fbcaacb4aa4f8dae3b6a1693297469, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/ae23a1fc22bb4929b3a0d6c4caf97e4c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b37f68dc79ce4ae3a0b795643ca5a625, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b421877b2983403ebfcfecb0aa677464, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/f63eb69171954e7394d1c5eaf55d1c88, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d2480a47eec54b138d9d8e2fd4f59193, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/dac61c04b8fe4e4f9ada2c0574d71aea, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/355de4c0fe4d49a6a9172624b499d426, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c4f725ec42104c2dbe94aa4e0d73fb28, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/31175a16f4d541d1a2c08d40c3e572b6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8f4eaed79a804d0eac7844cbd416ab69, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/52751f5eb9834a928e7bdaa4056161d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/e3cc4e729bbd4001afdef0bd6883b8c5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/1f9f6163659f44f5b898173ff17ce57e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/844a3f65d0f84890a24fcd8343b53f42, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/aa679491ebff4827b81299015d0bdbe8] to archive 2024-11-26T11:15:07,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:15:07,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8477a500078b4d3faa051bb78388210a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8477a500078b4d3faa051bb78388210a 2024-11-26T11:15:07,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7b88cc2bf5d94c9088525f50ad7fc96c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7b88cc2bf5d94c9088525f50ad7fc96c 2024-11-26T11:15:07,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/48fa92fb55ae42918a1db6fb221b1a74 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/48fa92fb55ae42918a1db6fb221b1a74 2024-11-26T11:15:07,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/df31f2af58fc4a8fa29b1a7976a3d83a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/df31f2af58fc4a8fa29b1a7976a3d83a 2024-11-26T11:15:07,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7dd0f2ef3a09410394398362f8984226 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/7dd0f2ef3a09410394398362f8984226 2024-11-26T11:15:07,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/389ab5ed5abf4a55baef98b04110cd5a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/389ab5ed5abf4a55baef98b04110cd5a 2024-11-26T11:15:07,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/44a205c267de411c884a5aa6441abbab to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/44a205c267de411c884a5aa6441abbab 2024-11-26T11:15:07,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/3e3685e7d62b4a6897b53983c3e84c4b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/3e3685e7d62b4a6897b53983c3e84c4b 2024-11-26T11:15:07,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8affdbca5eb34ff4aa5a7811eff6c71f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8affdbca5eb34ff4aa5a7811eff6c71f 2024-11-26T11:15:07,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/2911728143dc4d0cafe0a5b44f841d17 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/2911728143dc4d0cafe0a5b44f841d17 2024-11-26T11:15:07,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/691ee9ecb52c4fd299c3fcd33c3307e1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/691ee9ecb52c4fd299c3fcd33c3307e1 2024-11-26T11:15:07,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c6f55b250d8246048f5bbe5a040eb23a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c6f55b250d8246048f5bbe5a040eb23a 2024-11-26T11:15:07,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d7fbcaacb4aa4f8dae3b6a1693297469 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d7fbcaacb4aa4f8dae3b6a1693297469 2024-11-26T11:15:07,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/ae23a1fc22bb4929b3a0d6c4caf97e4c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/ae23a1fc22bb4929b3a0d6c4caf97e4c 2024-11-26T11:15:07,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b37f68dc79ce4ae3a0b795643ca5a625 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b37f68dc79ce4ae3a0b795643ca5a625 2024-11-26T11:15:07,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b421877b2983403ebfcfecb0aa677464 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/b421877b2983403ebfcfecb0aa677464 2024-11-26T11:15:07,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/f63eb69171954e7394d1c5eaf55d1c88 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/f63eb69171954e7394d1c5eaf55d1c88 2024-11-26T11:15:07,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d2480a47eec54b138d9d8e2fd4f59193 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/d2480a47eec54b138d9d8e2fd4f59193 2024-11-26T11:15:07,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/dac61c04b8fe4e4f9ada2c0574d71aea to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/dac61c04b8fe4e4f9ada2c0574d71aea 2024-11-26T11:15:07,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/355de4c0fe4d49a6a9172624b499d426 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/355de4c0fe4d49a6a9172624b499d426 2024-11-26T11:15:07,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c4f725ec42104c2dbe94aa4e0d73fb28 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/c4f725ec42104c2dbe94aa4e0d73fb28 2024-11-26T11:15:07,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/31175a16f4d541d1a2c08d40c3e572b6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/31175a16f4d541d1a2c08d40c3e572b6 2024-11-26T11:15:07,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8f4eaed79a804d0eac7844cbd416ab69 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/8f4eaed79a804d0eac7844cbd416ab69 2024-11-26T11:15:07,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/52751f5eb9834a928e7bdaa4056161d6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/52751f5eb9834a928e7bdaa4056161d6 2024-11-26T11:15:07,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/e3cc4e729bbd4001afdef0bd6883b8c5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/e3cc4e729bbd4001afdef0bd6883b8c5 2024-11-26T11:15:07,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/1f9f6163659f44f5b898173ff17ce57e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/1f9f6163659f44f5b898173ff17ce57e 2024-11-26T11:15:07,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/844a3f65d0f84890a24fcd8343b53f42 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/844a3f65d0f84890a24fcd8343b53f42 2024-11-26T11:15:07,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/aa679491ebff4827b81299015d0bdbe8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/aa679491ebff4827b81299015d0bdbe8 2024-11-26T11:15:07,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e9b6b4352ddc4fcbb04b62499b4e3325, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/caefcb08f9af423bb8726465bfc677db, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/13d66241bb214075a57871f408731223, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e09b6c7aeaa14353b91eac0bcd9e9a61, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/adf800f15980455d9b85ea5a89f6b66c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/04f9039c07f14d6f95f5d2dfd1305187, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/71b902c88783466497b06d4df49e6a37, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3eba2e3844f24cc9960e97fb8c6f506b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/1e0366b5b77c43c09088ad199203d823, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/2ec36bad84e1409a8b01d16914fd21dc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81b03f5c46e44b3a837729d054900fbe, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81124f9d658148ddb27bfa7adcf677df, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/339f4e7b01934f9ebc606e0ea781e14a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/02b7148518294fb69ef28187e4d4a8d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/61d5de79ec4f48448a30314b02b01856, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3a2c1160c98d41469f1a9bda732da5c7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/dba9fdc00cce4903b6e9c3e80b378288, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/fb197c9eff1d4d06bac71d59299bca97, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f047952991094aa9a54e93490ab6b318, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e916e93d50944a8995abe7092194c0f0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/c006af517f064e69bbf0dcdccb377deb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/31cbb2e92785491ab5c35f88f5ee9612, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e88fe70a85db48fe9177ab3020386882, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b4805ee0df254cc18b95566177411ee7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f3483b9a314a425f9e6cb9b5aa0543cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b0569277438c41a59411b945919ab7da] to archive 2024-11-26T11:15:07,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:15:07,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e9b6b4352ddc4fcbb04b62499b4e3325 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e9b6b4352ddc4fcbb04b62499b4e3325 2024-11-26T11:15:07,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/caefcb08f9af423bb8726465bfc677db to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/caefcb08f9af423bb8726465bfc677db 2024-11-26T11:15:07,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/13d66241bb214075a57871f408731223 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/13d66241bb214075a57871f408731223 2024-11-26T11:15:07,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e09b6c7aeaa14353b91eac0bcd9e9a61 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e09b6c7aeaa14353b91eac0bcd9e9a61 2024-11-26T11:15:07,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/adf800f15980455d9b85ea5a89f6b66c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/adf800f15980455d9b85ea5a89f6b66c 2024-11-26T11:15:07,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/04f9039c07f14d6f95f5d2dfd1305187 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/04f9039c07f14d6f95f5d2dfd1305187 2024-11-26T11:15:07,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/71b902c88783466497b06d4df49e6a37 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/71b902c88783466497b06d4df49e6a37 2024-11-26T11:15:07,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3eba2e3844f24cc9960e97fb8c6f506b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3eba2e3844f24cc9960e97fb8c6f506b 2024-11-26T11:15:07,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/1e0366b5b77c43c09088ad199203d823 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/1e0366b5b77c43c09088ad199203d823 2024-11-26T11:15:07,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/2ec36bad84e1409a8b01d16914fd21dc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/2ec36bad84e1409a8b01d16914fd21dc 2024-11-26T11:15:07,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81b03f5c46e44b3a837729d054900fbe to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81b03f5c46e44b3a837729d054900fbe 2024-11-26T11:15:07,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81124f9d658148ddb27bfa7adcf677df to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/81124f9d658148ddb27bfa7adcf677df 2024-11-26T11:15:07,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/339f4e7b01934f9ebc606e0ea781e14a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/339f4e7b01934f9ebc606e0ea781e14a 2024-11-26T11:15:07,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/02b7148518294fb69ef28187e4d4a8d6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/02b7148518294fb69ef28187e4d4a8d6 2024-11-26T11:15:07,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/9b47b2b5f1b34e0eaa7e8c7f2d3b2273 2024-11-26T11:15:07,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/61d5de79ec4f48448a30314b02b01856 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/61d5de79ec4f48448a30314b02b01856 2024-11-26T11:15:07,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3a2c1160c98d41469f1a9bda732da5c7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3a2c1160c98d41469f1a9bda732da5c7 2024-11-26T11:15:07,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/dba9fdc00cce4903b6e9c3e80b378288 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/dba9fdc00cce4903b6e9c3e80b378288 2024-11-26T11:15:07,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/fb197c9eff1d4d06bac71d59299bca97 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/fb197c9eff1d4d06bac71d59299bca97 2024-11-26T11:15:07,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f047952991094aa9a54e93490ab6b318 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f047952991094aa9a54e93490ab6b318 2024-11-26T11:15:07,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e916e93d50944a8995abe7092194c0f0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e916e93d50944a8995abe7092194c0f0 2024-11-26T11:15:07,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/c006af517f064e69bbf0dcdccb377deb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/c006af517f064e69bbf0dcdccb377deb 2024-11-26T11:15:07,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/0272aaa4d4fb4e5ba3e2be69b5ed0f45 2024-11-26T11:15:07,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/31cbb2e92785491ab5c35f88f5ee9612 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/31cbb2e92785491ab5c35f88f5ee9612 2024-11-26T11:15:07,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e88fe70a85db48fe9177ab3020386882 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/e88fe70a85db48fe9177ab3020386882 2024-11-26T11:15:07,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b4805ee0df254cc18b95566177411ee7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b4805ee0df254cc18b95566177411ee7 2024-11-26T11:15:07,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f3483b9a314a425f9e6cb9b5aa0543cd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/f3483b9a314a425f9e6cb9b5aa0543cd 2024-11-26T11:15:07,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b0569277438c41a59411b945919ab7da to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b0569277438c41a59411b945919ab7da 2024-11-26T11:15:07,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/22d50c5c70d8457890141427d77a8b73, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d03c403d85524888a5d78a601f471036, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/933c0d1c4625448eb7d5c2bc4738e71f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0d58730d21ca464ab15d6a110f41a634, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/523f1408a496401e993e4c7afeb3d1d0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d5c284724607479b9fc4a3976caa93e5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/83819a0e25fb4b6fb46e99a42c4e34cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/4e9ba4da81264ea1a414ea761d20c492, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/e16dab49fa0b48a48d085012881d2191, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/54034317b2ed4789ba4e7af4851372c9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/9e0fed06ed534a80a300a5148ff66c71, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/1203be2bb3694fe0a12a7eee38b205e5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c4ad1ae3254b2ea332de85f5ff3efd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/999933aac76e4d04bbba96d9a12181be, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/233780c7b03f4b4a8f631be270c73dcf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2ee95b04026644e4a446aea0461ef07d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/f0309a4195a34a148ba351796813d5ed, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0762d3b3bfc7499396b3adcfde72690f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3be8a3d02afc43f58611c117d96dfee8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c3b85ab16c4a94b1f777c0b7f95301, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a2fa36c6469d436f99c7b3b9256b168e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0027364cd5d04f88812ef2d333194227, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/90d2b79c97ca4c8380de5c6ad0e4246b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6b75069222ab494dbe38739a291095df, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2308f2a8b7c74af3a7974888f6b18644, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6d70bf6fed98440a86276edd87b08e35] to archive 2024-11-26T11:15:07,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:15:07,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a7b0cb6ee2c44e8cb6e7f10a7f4472bd 2024-11-26T11:15:07,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/22d50c5c70d8457890141427d77a8b73 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/22d50c5c70d8457890141427d77a8b73 2024-11-26T11:15:07,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d03c403d85524888a5d78a601f471036 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d03c403d85524888a5d78a601f471036 2024-11-26T11:15:07,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/933c0d1c4625448eb7d5c2bc4738e71f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/933c0d1c4625448eb7d5c2bc4738e71f 2024-11-26T11:15:07,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0d58730d21ca464ab15d6a110f41a634 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0d58730d21ca464ab15d6a110f41a634 2024-11-26T11:15:07,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/523f1408a496401e993e4c7afeb3d1d0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/523f1408a496401e993e4c7afeb3d1d0 2024-11-26T11:15:07,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d5c284724607479b9fc4a3976caa93e5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/d5c284724607479b9fc4a3976caa93e5 2024-11-26T11:15:07,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/83819a0e25fb4b6fb46e99a42c4e34cd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/83819a0e25fb4b6fb46e99a42c4e34cd 2024-11-26T11:15:07,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/4e9ba4da81264ea1a414ea761d20c492 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/4e9ba4da81264ea1a414ea761d20c492 2024-11-26T11:15:07,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/e16dab49fa0b48a48d085012881d2191 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/e16dab49fa0b48a48d085012881d2191 2024-11-26T11:15:07,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/54034317b2ed4789ba4e7af4851372c9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/54034317b2ed4789ba4e7af4851372c9 2024-11-26T11:15:07,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/9e0fed06ed534a80a300a5148ff66c71 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/9e0fed06ed534a80a300a5148ff66c71 2024-11-26T11:15:07,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/1203be2bb3694fe0a12a7eee38b205e5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/1203be2bb3694fe0a12a7eee38b205e5 2024-11-26T11:15:07,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c4ad1ae3254b2ea332de85f5ff3efd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c4ad1ae3254b2ea332de85f5ff3efd 2024-11-26T11:15:07,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/999933aac76e4d04bbba96d9a12181be to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/999933aac76e4d04bbba96d9a12181be 2024-11-26T11:15:07,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/233780c7b03f4b4a8f631be270c73dcf to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/233780c7b03f4b4a8f631be270c73dcf 2024-11-26T11:15:07,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2ee95b04026644e4a446aea0461ef07d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2ee95b04026644e4a446aea0461ef07d 2024-11-26T11:15:07,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/f0309a4195a34a148ba351796813d5ed to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/f0309a4195a34a148ba351796813d5ed 2024-11-26T11:15:07,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0762d3b3bfc7499396b3adcfde72690f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0762d3b3bfc7499396b3adcfde72690f 2024-11-26T11:15:07,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3be8a3d02afc43f58611c117d96dfee8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3be8a3d02afc43f58611c117d96dfee8 2024-11-26T11:15:07,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/fe4a00fd0d9e4ed8a2989f4e08fcbd01 2024-11-26T11:15:07,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c3b85ab16c4a94b1f777c0b7f95301 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/10c3b85ab16c4a94b1f777c0b7f95301 2024-11-26T11:15:07,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a2fa36c6469d436f99c7b3b9256b168e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a2fa36c6469d436f99c7b3b9256b168e 2024-11-26T11:15:07,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0027364cd5d04f88812ef2d333194227 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/0027364cd5d04f88812ef2d333194227 2024-11-26T11:15:07,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/90d2b79c97ca4c8380de5c6ad0e4246b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/90d2b79c97ca4c8380de5c6ad0e4246b 2024-11-26T11:15:07,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6b75069222ab494dbe38739a291095df to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6b75069222ab494dbe38739a291095df 2024-11-26T11:15:07,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2308f2a8b7c74af3a7974888f6b18644 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/2308f2a8b7c74af3a7974888f6b18644 2024-11-26T11:15:07,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6d70bf6fed98440a86276edd87b08e35 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/6d70bf6fed98440a86276edd87b08e35 2024-11-26T11:15:07,570 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=1 2024-11-26T11:15:07,570 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. 2024-11-26T11:15:07,571 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 77c23a8be60706f6017524015b3171ad: Waiting for close lock at 1732619706013Running coprocessor pre-close hooks at 1732619706013Disabling compacts and flushes for region at 1732619706013Disabling writes for close at 1732619706569 (+556 ms)Obtaining lock to block concurrent updates at 1732619706569Preparing flush snapshotting stores in 77c23a8be60706f6017524015b3171ad at 1732619706569Finished memstore snapshotting TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732619706570 (+1 ms)Flushing stores of TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad. at 1732619706570Flushing 77c23a8be60706f6017524015b3171ad/A: creating writer at 1732619706570Flushing 77c23a8be60706f6017524015b3171ad/A: appending metadata at 1732619706573 (+3 ms)Flushing 77c23a8be60706f6017524015b3171ad/A: closing flushed file at 1732619706573Flushing 77c23a8be60706f6017524015b3171ad/B: creating writer at 1732619706983 (+410 ms)Flushing 77c23a8be60706f6017524015b3171ad/B: appending metadata at 1732619706986 (+3 ms)Flushing 77c23a8be60706f6017524015b3171ad/B: closing flushed file at 1732619706987 (+1 ms)Flushing 77c23a8be60706f6017524015b3171ad/C: creating writer at 1732619707000 (+13 ms)Flushing 77c23a8be60706f6017524015b3171ad/C: appending metadata at 1732619707005 (+5 ms)Flushing 77c23a8be60706f6017524015b3171ad/C: closing flushed file at 1732619707005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a4b75f0: reopening flushed file at 1732619707416 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@767870c: reopening flushed file at 1732619707422 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5698d937: reopening flushed file at 1732619707428 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 77c23a8be60706f6017524015b3171ad in 866ms, sequenceid=428, compaction requested=true at 1732619707435 (+7 ms)Writing region close event to WAL at 1732619707566 (+131 ms)Running coprocessor post-close hooks at 1732619707570 (+4 ms)Closed at 1732619707570 2024-11-26T11:15:07,573 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:07,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=77c23a8be60706f6017524015b3171ad, regionState=CLOSED 2024-11-26T11:15:07,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:15:07,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-26T11:15:07,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 77c23a8be60706f6017524015b3171ad, server=7f3a3edaecc3,37499,1732619676667 in 1.7210 sec 2024-11-26T11:15:07,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-26T11:15:07,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=77c23a8be60706f6017524015b3171ad, UNASSIGN in 1.7290 sec 2024-11-26T11:15:07,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T11:15:07,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7360 sec 2024-11-26T11:15:07,585 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619707584"}]},"ts":"1732619707584"} 2024-11-26T11:15:07,586 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T11:15:07,586 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T11:15:07,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7510 sec 2024-11-26T11:15:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T11:15:07,993 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:07,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T11:15:08,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,004 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T11:15:08,006 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,012 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:08,027 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/recovered.edits] 2024-11-26T11:15:08,032 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/28479b66f9e7473fbf1b1214c81e1193 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/28479b66f9e7473fbf1b1214c81e1193 2024-11-26T11:15:08,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/5ea2cb67c4264272b745d562927e7a84 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/5ea2cb67c4264272b745d562927e7a84 2024-11-26T11:15:08,036 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/94557dbf9a7c46cf8268173e7341ec9d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/A/94557dbf9a7c46cf8268173e7341ec9d 2024-11-26T11:15:08,041 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3c764192eb4a4dff93d7c55d3a609c4d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/3c764192eb4a4dff93d7c55d3a609c4d 2024-11-26T11:15:08,043 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/695d75f1d3894c1d8670009f6348b528 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/695d75f1d3894c1d8670009f6348b528 2024-11-26T11:15:08,044 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b15f84e3676d4b48bf4c0a91465269b3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/B/b15f84e3676d4b48bf4c0a91465269b3 2024-11-26T11:15:08,047 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/3ac4a4eb9fee4ad5a1d34b52f43e6abc 2024-11-26T11:15:08,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/5cf80f9fcaed457e8830020fdec6face to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/5cf80f9fcaed457e8830020fdec6face 2024-11-26T11:15:08,051 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a8f7136d672946ed8fd4e71fb4d54868 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/C/a8f7136d672946ed8fd4e71fb4d54868 2024-11-26T11:15:08,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/recovered.edits/431.seqid to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad/recovered.edits/431.seqid 2024-11-26T11:15:08,056 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/77c23a8be60706f6017524015b3171ad 2024-11-26T11:15:08,056 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T11:15:08,064 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T11:15:08,077 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T11:15:08,112 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T11:15:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T11:15:08,114 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,114 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T11:15:08,115 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732619708114"}]},"ts":"9223372036854775807"} 2024-11-26T11:15:08,119 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T11:15:08,119 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 77c23a8be60706f6017524015b3171ad, NAME => 'TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T11:15:08,120 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T11:15:08,120 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732619708120"}]},"ts":"9223372036854775807"} 2024-11-26T11:15:08,123 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T11:15:08,124 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 126 msec 2024-11-26T11:15:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T11:15:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T11:15:08,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:08,339 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_181182078_22 at /127.0.0.1:52300 [Waiting for operation #287] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7f3a3edaecc3:37499-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1301850378_22 at /127.0.0.1:46058 [Waiting for operation #277] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 334) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2867 (was 3495) 2024-11-26T11:15:08,350 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=2867 2024-11-26T11:15:08,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:15:08,352 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:15:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:08,354 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T11:15:08,355 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:08,355 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-26T11:15:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T11:15:08,356 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T11:15:08,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741930_1106 (size=1061) 2024-11-26T11:15:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T11:15:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T11:15:08,770 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:15:08,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741931_1107 (size=53) 2024-11-26T11:15:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T11:15:09,183 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:15:09,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b258fbd035e77c2729be5c41f0c32984, disabling compactions & flushes 2024-11-26T11:15:09,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. after waiting 0 ms 2024-11-26T11:15:09,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b258fbd035e77c2729be5c41f0c32984: Waiting for close lock at 1732619709183Disabling compacts and flushes for region at 1732619709183Disabling writes for close at 1732619709184 (+1 ms)Writing region close event to WAL at 1732619709184Closed at 1732619709184 2024-11-26T11:15:09,187 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T11:15:09,188 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732619709187"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732619709187"}]},"ts":"1732619709187"} 2024-11-26T11:15:09,194 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T11:15:09,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T11:15:09,196 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619709196"}]},"ts":"1732619709196"} 2024-11-26T11:15:09,198 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T11:15:09,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, ASSIGN}] 2024-11-26T11:15:09,200 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, ASSIGN 2024-11-26T11:15:09,201 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, ASSIGN; state=OFFLINE, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=false 2024-11-26T11:15:09,352 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:09,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, ASSIGN because future has completed 2024-11-26T11:15:09,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:15:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T11:15:09,524 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,524 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:15:09,525 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,525 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:15:09,526 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,526 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,529 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,531 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:09,532 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b258fbd035e77c2729be5c41f0c32984 columnFamilyName A 2024-11-26T11:15:09,532 DEBUG [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:09,533 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(327): Store=b258fbd035e77c2729be5c41f0c32984/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:09,533 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,534 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:09,534 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b258fbd035e77c2729be5c41f0c32984 columnFamilyName B 2024-11-26T11:15:09,534 DEBUG [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:09,535 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(327): Store=b258fbd035e77c2729be5c41f0c32984/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:09,535 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,536 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:09,536 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b258fbd035e77c2729be5c41f0c32984 columnFamilyName C 2024-11-26T11:15:09,536 DEBUG [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:09,536 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(327): Store=b258fbd035e77c2729be5c41f0c32984/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:09,537 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,537 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,537 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,538 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,539 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,539 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,539 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:15:09,540 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,542 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:15:09,542 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened b258fbd035e77c2729be5c41f0c32984; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75428576, jitterRate=0.12397336959838867}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:15:09,542 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:09,543 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for b258fbd035e77c2729be5c41f0c32984: Running coprocessor pre-open hook at 1732619709526Writing region info on filesystem at 1732619709526Initializing all the Stores at 1732619709528 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619709528Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619709529 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619709529Cleaning up temporary data from old regions at 1732619709539 (+10 ms)Running coprocessor post-open hooks at 1732619709542 (+3 ms)Region opened successfully at 1732619709543 (+1 ms) 2024-11-26T11:15:09,544 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., pid=28, masterSystemTime=1732619709516 2024-11-26T11:15:09,546 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,546 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:09,547 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:09,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:15:09,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-26T11:15:09,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 in 188 msec 2024-11-26T11:15:09,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-26T11:15:09,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, ASSIGN in 355 msec 2024-11-26T11:15:09,556 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T11:15:09,556 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619709556"}]},"ts":"1732619709556"} 2024-11-26T11:15:09,557 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T11:15:09,558 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T11:15:09,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2060 sec 2024-11-26T11:15:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T11:15:10,505 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:10,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:10,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:10,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:10,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a2829d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:10,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:10,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3df773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:10,518 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:10,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:10,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:15:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:15:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44e20f97 2024-11-26T11:15:10,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:15:10,526 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:15:10,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:15:10,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:15:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:10,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741932_1108 (size=1097) 2024-11-26T11:15:10,958 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T11:15:10,959 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T11:15:10,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:15:10,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, REOPEN/MOVE}] 2024-11-26T11:15:10,974 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, REOPEN/MOVE 2024-11-26T11:15:10,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:10,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, REOPEN/MOVE because future has completed 2024-11-26T11:15:10,977 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:15:10,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:15:11,130 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,130 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:15:11,130 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing b258fbd035e77c2729be5c41f0c32984, disabling compactions & flushes 2024-11-26T11:15:11,130 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,130 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,130 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. after waiting 0 ms 2024-11-26T11:15:11,130 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,140 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T11:15:11,141 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,141 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for b258fbd035e77c2729be5c41f0c32984: Waiting for close lock at 1732619711130Running coprocessor pre-close hooks at 1732619711130Disabling compacts and flushes for region at 1732619711130Disabling writes for close at 1732619711130Writing region close event to WAL at 1732619711131 (+1 ms)Running coprocessor post-close hooks at 1732619711141 (+10 ms)Closed at 1732619711141 2024-11-26T11:15:11,142 WARN [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: b258fbd035e77c2729be5c41f0c32984 to self. 2024-11-26T11:15:11,145 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,146 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=CLOSED 2024-11-26T11:15:11,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:15:11,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-26T11:15:11,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 in 173 msec 2024-11-26T11:15:11,153 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, REOPEN/MOVE; state=CLOSED, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=true 2024-11-26T11:15:11,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, REOPEN/MOVE because future has completed 2024-11-26T11:15:11,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:15:11,472 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,473 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:15:11,473 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,473 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:15:11,474 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,474 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,480 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,480 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:11,546 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b258fbd035e77c2729be5c41f0c32984 columnFamilyName A 2024-11-26T11:15:11,551 DEBUG [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:11,552 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(327): Store=b258fbd035e77c2729be5c41f0c32984/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:11,553 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,554 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:11,554 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b258fbd035e77c2729be5c41f0c32984 columnFamilyName B 2024-11-26T11:15:11,554 DEBUG [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:11,554 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(327): Store=b258fbd035e77c2729be5c41f0c32984/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:11,554 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,555 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:11,555 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b258fbd035e77c2729be5c41f0c32984 columnFamilyName C 2024-11-26T11:15:11,555 DEBUG [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:11,555 INFO [StoreOpener-b258fbd035e77c2729be5c41f0c32984-1 {}] regionserver.HStore(327): Store=b258fbd035e77c2729be5c41f0c32984/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:11,556 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,556 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,556 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,557 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,558 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,558 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,559 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:15:11,560 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,560 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened b258fbd035e77c2729be5c41f0c32984; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62763003, jitterRate=-0.06475837528705597}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:15:11,561 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,562 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for b258fbd035e77c2729be5c41f0c32984: Running coprocessor pre-open hook at 1732619711474Writing region info on filesystem at 1732619711474Initializing all the Stores at 1732619711476 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619711476Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619711476Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619711476Cleaning up temporary data from old regions at 1732619711558 (+82 ms)Running coprocessor post-open hooks at 1732619711561 (+3 ms)Region opened successfully at 1732619711562 (+1 ms) 2024-11-26T11:15:11,563 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., pid=33, masterSystemTime=1732619711464 2024-11-26T11:15:11,565 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,565 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=OPEN, openSeqNum=5, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:15:11,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-26T11:15:11,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 in 258 msec 2024-11-26T11:15:11,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-26T11:15:11,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, REOPEN/MOVE in 597 msec 2024-11-26T11:15:11,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T11:15:11,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 607 msec 2024-11-26T11:15:11,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0410 sec 2024-11-26T11:15:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T11:15:11,581 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:11,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,583 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d38218e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,585 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb77e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e32af12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,591 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308c21d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,593 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a7a7c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,595 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9f89ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674c6888, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22547ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,600 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@105e7978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,602 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2664cd82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,604 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2574981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,606 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6817d8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ed7830, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eadcc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0eac94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:11,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:11,613 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b0167b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:11,615 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:11,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e246ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:11,617 DEBUG [Thread-568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,617 DEBUG [Thread-570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,617 DEBUG [Thread-575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:11,617 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,617 DEBUG [Thread-573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,617 DEBUG [Thread-562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,618 DEBUG [Thread-570 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,618 DEBUG [Thread-570 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-26T11:15:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,620 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T11:15:11,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:11,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,627 DEBUG [Thread-564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,631 DEBUG [Thread-566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,632 DEBUG [Thread-577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:11,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:11,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:15:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:11,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619771689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619771689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619711681, max timestamp=1732619711694Num uniques 859; 2024-11-26T11:15:11,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:15:11,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619711681, max timestamp=1732619711694Num uniques 859; 2024-11-26T11:15:11,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:15:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619711681, max timestamp=1732619711694Num uniques 859; 2024-11-26T11:15:11,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:15:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bb99f8405d624ef7b3d460b13f08d46b_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619711635/Put/seqid=0 2024-11-26T11:15:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T11:15:11,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741933_1109 (size=17034) 2024-11-26T11:15:11,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:11,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bb99f8405d624ef7b3d460b13f08d46b_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bb99f8405d624ef7b3d460b13f08d46b_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:11,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9a83c28bb53e497cab950a4719ed1b75, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:11,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619771744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9a83c28bb53e497cab950a4719ed1b75 is 175, key is test_row_0/A:col10/1732619711635/Put/seqid=0 2024-11-26T11:15:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619771747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619771748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741934_1110 (size=48139) 2024-11-26T11:15:11,769 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9a83c28bb53e497cab950a4719ed1b75 2024-11-26T11:15:11,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:11,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:11,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:11,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619771804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619771804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/3d7c8126fa674112be85f07f3d3211e0 is 50, key is test_row_0/B:col10/1732619711635/Put/seqid=0 2024-11-26T11:15:11,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741935_1111 (size=12001) 2024-11-26T11:15:11,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/3d7c8126fa674112be85f07f3d3211e0 2024-11-26T11:15:11,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619771863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619771864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619771864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:11,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ad8ed8727f2548a8be6ad2437b3983e8 is 50, key is test_row_0/C:col10/1732619711635/Put/seqid=0 2024-11-26T11:15:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741936_1112 (size=12001) 2024-11-26T11:15:11,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ad8ed8727f2548a8be6ad2437b3983e8 2024-11-26T11:15:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9a83c28bb53e497cab950a4719ed1b75 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75 2024-11-26T11:15:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T11:15:11,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:11,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:11,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75, entries=250, sequenceid=15, filesize=47.0 K 2024-11-26T11:15:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:11,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/3d7c8126fa674112be85f07f3d3211e0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3d7c8126fa674112be85f07f3d3211e0 2024-11-26T11:15:11,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3d7c8126fa674112be85f07f3d3211e0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T11:15:11,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ad8ed8727f2548a8be6ad2437b3983e8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ad8ed8727f2548a8be6ad2437b3983e8 2024-11-26T11:15:11,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ad8ed8727f2548a8be6ad2437b3983e8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T11:15:11,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b258fbd035e77c2729be5c41f0c32984 in 312ms, sequenceid=15, compaction requested=false 2024-11-26T11:15:11,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:12,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T11:15:12,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:12,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:12,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:12,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:12,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:12,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:12,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112621a15005d8054c7daeb4d33d02a64972_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619711733/Put/seqid=0 2024-11-26T11:15:12,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619772047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741937_1113 (size=14594) 2024-11-26T11:15:12,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619772052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619772074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619772075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619772075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619772152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619772163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T11:15:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619772373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619772373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619772393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619772393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619772393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:12,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112621a15005d8054c7daeb4d33d02a64972_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112621a15005d8054c7daeb4d33d02a64972_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:12,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/142169c966a945a0aaaf78f3f43307b8, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:12,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/142169c966a945a0aaaf78f3f43307b8 is 175, key is test_row_0/A:col10/1732619711733/Put/seqid=0 2024-11-26T11:15:12,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741938_1114 (size=39549) 2024-11-26T11:15:12,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:12,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:12,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619772683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619772683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:12,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T11:15:12,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:12,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:12,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/142169c966a945a0aaaf78f3f43307b8 2024-11-26T11:15:12,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/1000f4abf6f5450cafccf841a4c1d362 is 50, key is test_row_0/B:col10/1732619711733/Put/seqid=0 2024-11-26T11:15:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741939_1115 (size=12001) 2024-11-26T11:15:12,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/1000f4abf6f5450cafccf841a4c1d362 2024-11-26T11:15:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619772903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619772903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619772917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:12,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/948f3a6bd3da479e9c8e5081c335ab8f is 50, key is test_row_0/C:col10/1732619711733/Put/seqid=0 2024-11-26T11:15:12,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741940_1116 (size=12001) 2024-11-26T11:15:12,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/948f3a6bd3da479e9c8e5081c335ab8f 2024-11-26T11:15:12,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/142169c966a945a0aaaf78f3f43307b8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8 2024-11-26T11:15:12,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8, entries=200, sequenceid=51, filesize=38.6 K 2024-11-26T11:15:12,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/1000f4abf6f5450cafccf841a4c1d362 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1000f4abf6f5450cafccf841a4c1d362 2024-11-26T11:15:12,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1000f4abf6f5450cafccf841a4c1d362, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T11:15:12,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/948f3a6bd3da479e9c8e5081c335ab8f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/948f3a6bd3da479e9c8e5081c335ab8f 2024-11-26T11:15:12,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/948f3a6bd3da479e9c8e5081c335ab8f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T11:15:12,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for b258fbd035e77c2729be5c41f0c32984 in 964ms, sequenceid=51, compaction requested=false 2024-11-26T11:15:12,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:13,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T11:15:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:13,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:13,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126580d2deaa2e149909998b2941c591d39_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619712021/Put/seqid=0 2024-11-26T11:15:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741941_1117 (size=12154) 2024-11-26T11:15:13,103 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:15:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:13,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:13,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619713204, max timestamp=1732619713237Num uniques 859; 2024-11-26T11:15:13,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:13,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619713204, max timestamp=1732619713237Num uniques 859; 2024-11-26T11:15:13,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:13,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619713204, max timestamp=1732619713237Num uniques 859; 2024-11-26T11:15:13,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:13,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619773290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619773292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619773402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619773403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:13,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126580d2deaa2e149909998b2941c591d39_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126580d2deaa2e149909998b2941c591d39_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/5f220d641b4c4e2598251486e108bcf8, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/5f220d641b4c4e2598251486e108bcf8 is 175, key is test_row_0/A:col10/1732619712021/Put/seqid=0 2024-11-26T11:15:13,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741942_1118 (size=30955) 2024-11-26T11:15:13,476 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/5f220d641b4c4e2598251486e108bcf8 2024-11-26T11:15:13,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/b32b06e8cd7d4023a75b2bbb0a7e3b80 is 50, key is test_row_0/B:col10/1732619712021/Put/seqid=0 2024-11-26T11:15:13,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741943_1119 (size=12001) 2024-11-26T11:15:13,496 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/b32b06e8cd7d4023a75b2bbb0a7e3b80 2024-11-26T11:15:13,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/4c273c09bbf14609b57a1fa4fe9a6e49 is 50, key is test_row_0/C:col10/1732619712021/Put/seqid=0 2024-11-26T11:15:13,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741944_1120 (size=12001) 2024-11-26T11:15:13,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/4c273c09bbf14609b57a1fa4fe9a6e49 2024-11-26T11:15:13,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/5f220d641b4c4e2598251486e108bcf8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8 2024-11-26T11:15:13,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8, entries=150, sequenceid=61, filesize=30.2 K 2024-11-26T11:15:13,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/b32b06e8cd7d4023a75b2bbb0a7e3b80 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/b32b06e8cd7d4023a75b2bbb0a7e3b80 2024-11-26T11:15:13,552 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/b32b06e8cd7d4023a75b2bbb0a7e3b80, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T11:15:13,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/4c273c09bbf14609b57a1fa4fe9a6e49 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4c273c09bbf14609b57a1fa4fe9a6e49 2024-11-26T11:15:13,558 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4c273c09bbf14609b57a1fa4fe9a6e49, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T11:15:13,559 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b258fbd035e77c2729be5c41f0c32984 in 544ms, sequenceid=61, compaction requested=true 2024-11-26T11:15:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-26T11:15:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-26T11:15:13,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-26T11:15:13,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-26T11:15:13,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.9470 sec 2024-11-26T11:15:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:13,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-26T11:15:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:13,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126454c5ef34dd6425c969250f1d4eb83b1_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:13,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741945_1121 (size=14594) 2024-11-26T11:15:13,653 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:13,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126454c5ef34dd6425c969250f1d4eb83b1_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126454c5ef34dd6425c969250f1d4eb83b1_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:13,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:13,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42 is 175, key is test_row_0/A:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:13,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741946_1122 (size=39549) 2024-11-26T11:15:13,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619773677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619773677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T11:15:13,763 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:13,763 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-26T11:15:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:13,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:13,766 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:13,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619773793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619773793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619773912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:13,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:13,918 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in b258fbd035e77c2729be5c41f0c32984 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:13,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619773922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,924 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in b258fbd035e77c2729be5c41f0c32984 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619773933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:13,936 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in b258fbd035e77c2729be5c41f0c32984 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619774002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619774002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,067 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42 2024-11-26T11:15:14,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/3e43e08efc8e4eaf86cce8590665c5cd is 50, key is test_row_0/B:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:14,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741947_1123 (size=12001) 2024-11-26T11:15:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:14,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619774313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619774313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:14,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/3e43e08efc8e4eaf86cce8590665c5cd 2024-11-26T11:15:14,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/41e7f9ea68414a668a66c32dc0b1f3ea is 50, key is test_row_0/C:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:14,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741948_1124 (size=12001) 2024-11-26T11:15:14,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619774823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619774823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:14,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:14,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/41e7f9ea68414a668a66c32dc0b1f3ea 2024-11-26T11:15:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42 2024-11-26T11:15:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:14,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42, entries=200, sequenceid=97, filesize=38.6 K 2024-11-26T11:15:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/3e43e08efc8e4eaf86cce8590665c5cd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3e43e08efc8e4eaf86cce8590665c5cd 2024-11-26T11:15:14,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3e43e08efc8e4eaf86cce8590665c5cd, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T11:15:14,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/41e7f9ea68414a668a66c32dc0b1f3ea as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/41e7f9ea68414a668a66c32dc0b1f3ea 2024-11-26T11:15:14,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/41e7f9ea68414a668a66c32dc0b1f3ea, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T11:15:14,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for b258fbd035e77c2729be5c41f0c32984 in 1307ms, sequenceid=97, compaction requested=true 2024-11-26T11:15:14,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:14,924 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:14,925 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:14,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:14,927 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:14,927 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:14,928 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,928 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=154.5 K 2024-11-26T11:15:14,928 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,928 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42] 2024-11-26T11:15:14,929 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:14,929 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:14,929 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,929 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3d7c8126fa674112be85f07f3d3211e0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1000f4abf6f5450cafccf841a4c1d362, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/b32b06e8cd7d4023a75b2bbb0a7e3b80, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3e43e08efc8e4eaf86cce8590665c5cd] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=46.9 K 2024-11-26T11:15:14,929 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a83c28bb53e497cab950a4719ed1b75, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732619711632 2024-11-26T11:15:14,929 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d7c8126fa674112be85f07f3d3211e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732619711635 2024-11-26T11:15:14,930 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142169c966a945a0aaaf78f3f43307b8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732619711733 2024-11-26T11:15:14,930 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1000f4abf6f5450cafccf841a4c1d362, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732619711733 2024-11-26T11:15:14,930 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f220d641b4c4e2598251486e108bcf8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732619712021 2024-11-26T11:15:14,930 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b32b06e8cd7d4023a75b2bbb0a7e3b80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732619712021 2024-11-26T11:15:14,930 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f6b2d9c5764c9cb7acc1ddbc4d5a42, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619713269 2024-11-26T11:15:14,931 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e43e08efc8e4eaf86cce8590665c5cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619713269 2024-11-26T11:15:14,941 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:14,943 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:14,943 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/2e2053a5fd574e5dbecc3a261451ca21 is 50, key is test_row_0/B:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:14,948 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268feb518d0fe0434aa7edaf8a6f0eef17_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:14,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741949_1125 (size=12139) 2024-11-26T11:15:14,955 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268feb518d0fe0434aa7edaf8a6f0eef17_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:14,955 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268feb518d0fe0434aa7edaf8a6f0eef17_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741950_1126 (size=4469) 2024-11-26T11:15:14,963 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#106 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:14,964 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8fb487a9bc8b403fa13786627f0e0901 is 175, key is test_row_0/A:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:14,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741951_1127 (size=31093) 2024-11-26T11:15:14,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T11:15:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:14,990 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264c5292e8619d448684629f6f561ba52c_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619713650/Put/seqid=0 2024-11-26T11:15:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741952_1128 (size=12154) 2024-11-26T11:15:15,358 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/2e2053a5fd574e5dbecc3a261451ca21 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/2e2053a5fd574e5dbecc3a261451ca21 2024-11-26T11:15:15,363 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into 2e2053a5fd574e5dbecc3a261451ca21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:15,363 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:15,363 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=12, startTime=1732619714925; duration=0sec 2024-11-26T11:15:15,363 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:15,363 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:15,363 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:15,366 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:15,366 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:15,366 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:15,366 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ad8ed8727f2548a8be6ad2437b3983e8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/948f3a6bd3da479e9c8e5081c335ab8f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4c273c09bbf14609b57a1fa4fe9a6e49, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/41e7f9ea68414a668a66c32dc0b1f3ea] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=46.9 K 2024-11-26T11:15:15,367 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8ed8727f2548a8be6ad2437b3983e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732619711635 2024-11-26T11:15:15,367 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 948f3a6bd3da479e9c8e5081c335ab8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732619711733 2024-11-26T11:15:15,367 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c273c09bbf14609b57a1fa4fe9a6e49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732619712021 2024-11-26T11:15:15,368 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 41e7f9ea68414a668a66c32dc0b1f3ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619713269 2024-11-26T11:15:15,373 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8fb487a9bc8b403fa13786627f0e0901 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8fb487a9bc8b403fa13786627f0e0901 2024-11-26T11:15:15,378 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into 8fb487a9bc8b403fa13786627f0e0901(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:15,378 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:15,379 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=12, startTime=1732619714924; duration=0sec 2024-11-26T11:15:15,379 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:15,379 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:15,380 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:15,380 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ae04ee4be321466299b6ad7fdf59382d is 50, key is test_row_0/C:col10/1732619713269/Put/seqid=0 2024-11-26T11:15:15,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741953_1129 (size=12139) 2024-11-26T11:15:15,390 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ae04ee4be321466299b6ad7fdf59382d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ae04ee4be321466299b6ad7fdf59382d 2024-11-26T11:15:15,395 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into ae04ee4be321466299b6ad7fdf59382d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:15,395 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:15,395 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=12, startTime=1732619714925; duration=0sec 2024-11-26T11:15:15,395 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:15,395 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:15,412 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264c5292e8619d448684629f6f561ba52c_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264c5292e8619d448684629f6f561ba52c_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/bf1ce246a8d94998b5a0a5ff44ca2900, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/bf1ce246a8d94998b5a0a5ff44ca2900 is 175, key is test_row_0/A:col10/1732619713650/Put/seqid=0 2024-11-26T11:15:15,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741954_1130 (size=30955) 2024-11-26T11:15:15,427 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/bf1ce246a8d94998b5a0a5ff44ca2900 2024-11-26T11:15:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/1e5017d792744ffa8c3db049bd2045b8 is 50, key is test_row_0/B:col10/1732619713650/Put/seqid=0 2024-11-26T11:15:15,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741955_1131 (size=12001) 2024-11-26T11:15:15,455 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/1e5017d792744ffa8c3db049bd2045b8 2024-11-26T11:15:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/17b9835a42df4d5890fa0ba1225f237c is 50, key is test_row_0/C:col10/1732619713650/Put/seqid=0 2024-11-26T11:15:15,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741956_1132 (size=12001) 2024-11-26T11:15:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:15,875 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/17b9835a42df4d5890fa0ba1225f237c 2024-11-26T11:15:15,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619715859, max timestamp=1732619715877Num uniques 859; 2024-11-26T11:15:15,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:15,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619715859, max timestamp=1732619715877Num uniques 859; 2024-11-26T11:15:15,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:15,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619715859, max timestamp=1732619715877Num uniques 859; 2024-11-26T11:15:15,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:15,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/bf1ce246a8d94998b5a0a5ff44ca2900 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900 2024-11-26T11:15:15,892 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900, entries=150, sequenceid=107, filesize=30.2 K 2024-11-26T11:15:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/1e5017d792744ffa8c3db049bd2045b8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1e5017d792744ffa8c3db049bd2045b8 2024-11-26T11:15:15,904 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1e5017d792744ffa8c3db049bd2045b8, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T11:15:15,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/17b9835a42df4d5890fa0ba1225f237c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/17b9835a42df4d5890fa0ba1225f237c 2024-11-26T11:15:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:15,916 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/17b9835a42df4d5890fa0ba1225f237c, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T11:15:15,919 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=114.05 KB/116790 for b258fbd035e77c2729be5c41f0c32984 in 928ms, sequenceid=107, compaction requested=false 2024-11-26T11:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-26T11:15:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:15,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-26T11:15:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-26T11:15:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:15,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:15,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-26T11:15:15,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-11-26T11:15:15,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1640 sec 2024-11-26T11:15:15,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264084356fe468495f89144a572b531b9f_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:15,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619775946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619775950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619775952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741957_1133 (size=14794) 2024-11-26T11:15:15,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:15,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619775962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:15,969 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in b258fbd035e77c2729be5c41f0c32984 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:15,978 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264084356fe468495f89144a572b531b9f_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264084356fe468495f89144a572b531b9f_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:15,979 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/6de2004e207e4073a8a8a36b4910393d, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:15,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/6de2004e207e4073a8a8a36b4910393d is 175, key is test_row_0/A:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:15,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741958_1134 (size=39749) 2024-11-26T11:15:15,992 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/6de2004e207e4073a8a8a36b4910393d 2024-11-26T11:15:16,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/177d30f9cca542b48b32af711e066fbb is 50, key is test_row_0/B:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:16,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741959_1135 (size=12151) 2024-11-26T11:15:16,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/177d30f9cca542b48b32af711e066fbb 2024-11-26T11:15:16,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/46a52cb47ede4b88bf5e53326d4ef6e2 is 50, key is test_row_0/C:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:16,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741960_1136 (size=12151) 2024-11-26T11:15:16,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/46a52cb47ede4b88bf5e53326d4ef6e2 2024-11-26T11:15:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619776064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619776064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619776064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619776066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/6de2004e207e4073a8a8a36b4910393d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d 2024-11-26T11:15:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d, entries=200, sequenceid=143, filesize=38.8 K 2024-11-26T11:15:16,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/177d30f9cca542b48b32af711e066fbb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/177d30f9cca542b48b32af711e066fbb 2024-11-26T11:15:16,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/177d30f9cca542b48b32af711e066fbb, entries=150, sequenceid=143, filesize=11.9 K 2024-11-26T11:15:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/46a52cb47ede4b88bf5e53326d4ef6e2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46a52cb47ede4b88bf5e53326d4ef6e2 2024-11-26T11:15:16,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46a52cb47ede4b88bf5e53326d4ef6e2, entries=150, sequenceid=143, filesize=11.9 K 2024-11-26T11:15:16,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for b258fbd035e77c2729be5c41f0c32984 in 180ms, sequenceid=143, compaction requested=true 2024-11-26T11:15:16,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:16,100 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:16,101 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:16,103 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:16,103 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:16,103 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:16,103 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8fb487a9bc8b403fa13786627f0e0901, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=99.4 K 2024-11-26T11:15:16,103 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:16,103 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8fb487a9bc8b403fa13786627f0e0901, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d] 2024-11-26T11:15:16,103 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:16,104 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:16,104 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:16,104 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fb487a9bc8b403fa13786627f0e0901, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619713269 2024-11-26T11:15:16,104 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/2e2053a5fd574e5dbecc3a261451ca21, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1e5017d792744ffa8c3db049bd2045b8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/177d30f9cca542b48b32af711e066fbb] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=35.4 K 2024-11-26T11:15:16,104 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2053a5fd574e5dbecc3a261451ca21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619713269 2024-11-26T11:15:16,104 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1ce246a8d94998b5a0a5ff44ca2900, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732619713626 2024-11-26T11:15:16,105 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5017d792744ffa8c3db049bd2045b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732619713626 2024-11-26T11:15:16,105 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de2004e207e4073a8a8a36b4910393d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619715889 2024-11-26T11:15:16,107 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 177d30f9cca542b48b32af711e066fbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619715889 2024-11-26T11:15:16,116 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:16,117 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/f11abfd03ece4ebfa1091c97be223a1d is 50, key is test_row_0/B:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:16,129 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:16,137 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411266c0934fa7ec440c0aad0a15065de7d51_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:16,141 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411266c0934fa7ec440c0aad0a15065de7d51_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:16,141 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266c0934fa7ec440c0aad0a15065de7d51_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741961_1137 (size=12391) 2024-11-26T11:15:16,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741962_1138 (size=4469) 2024-11-26T11:15:16,172 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#115 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:16,172 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/f85340627413484e9144d3a941fa04f7 is 175, key is test_row_0/A:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:16,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741963_1139 (size=31345) 2024-11-26T11:15:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:16,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T11:15:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:16,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:16,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:16,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267a01bc9668cd49bc8fdb86cb93e69a03_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619715950/Put/seqid=0 2024-11-26T11:15:16,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741964_1140 (size=14794) 2024-11-26T11:15:16,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619776296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619776297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619776297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619776298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619776413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619776413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619776413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619776413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,586 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/f11abfd03ece4ebfa1091c97be223a1d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f11abfd03ece4ebfa1091c97be223a1d 2024-11-26T11:15:16,589 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/f85340627413484e9144d3a941fa04f7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/f85340627413484e9144d3a941fa04f7 2024-11-26T11:15:16,592 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into f11abfd03ece4ebfa1091c97be223a1d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:16,592 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:16,592 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619716101; duration=0sec 2024-11-26T11:15:16,592 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:16,592 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:16,592 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:16,594 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:16,594 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:16,594 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:16,594 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ae04ee4be321466299b6ad7fdf59382d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/17b9835a42df4d5890fa0ba1225f237c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46a52cb47ede4b88bf5e53326d4ef6e2] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=35.4 K 2024-11-26T11:15:16,595 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ae04ee4be321466299b6ad7fdf59382d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619713269 2024-11-26T11:15:16,595 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 17b9835a42df4d5890fa0ba1225f237c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732619713626 2024-11-26T11:15:16,595 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a52cb47ede4b88bf5e53326d4ef6e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619715889 2024-11-26T11:15:16,595 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into f85340627413484e9144d3a941fa04f7(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:16,595 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:16,595 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619716100; duration=0sec 2024-11-26T11:15:16,596 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:16,596 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:16,607 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:16,608 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ce2e196a84c54848aa27c8b4f70714f2 is 50, key is test_row_0/C:col10/1732619715889/Put/seqid=0 2024-11-26T11:15:16,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741965_1141 (size=12391) 2024-11-26T11:15:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619776623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619776623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619776623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619776623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,632 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/ce2e196a84c54848aa27c8b4f70714f2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ce2e196a84c54848aa27c8b4f70714f2 2024-11-26T11:15:16,640 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into ce2e196a84c54848aa27c8b4f70714f2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:16,640 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:16,640 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619716101; duration=0sec 2024-11-26T11:15:16,640 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:16,640 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:16,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:16,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267a01bc9668cd49bc8fdb86cb93e69a03_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267a01bc9668cd49bc8fdb86cb93e69a03_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:16,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8126293f831b4be5a774ca0865ff4ca5, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:16,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8126293f831b4be5a774ca0865ff4ca5 is 175, key is test_row_0/A:col10/1732619715950/Put/seqid=0 2024-11-26T11:15:16,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741966_1142 (size=39749) 2024-11-26T11:15:16,736 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8126293f831b4be5a774ca0865ff4ca5 2024-11-26T11:15:16,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/7a832c24c08e40cca5da36c7edc331b3 is 50, key is test_row_0/B:col10/1732619715950/Put/seqid=0 2024-11-26T11:15:16,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741967_1143 (size=12151) 2024-11-26T11:15:16,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/7a832c24c08e40cca5da36c7edc331b3 2024-11-26T11:15:16,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/1a73cee5cbe5478ba58dca90d1110b23 is 50, key is test_row_0/C:col10/1732619715950/Put/seqid=0 2024-11-26T11:15:16,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741968_1144 (size=12151) 2024-11-26T11:15:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619776933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619776934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619776934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:16,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619776943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/1a73cee5cbe5478ba58dca90d1110b23 2024-11-26T11:15:17,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8126293f831b4be5a774ca0865ff4ca5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5 2024-11-26T11:15:17,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5, entries=200, sequenceid=156, filesize=38.8 K 2024-11-26T11:15:17,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/7a832c24c08e40cca5da36c7edc331b3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/7a832c24c08e40cca5da36c7edc331b3 2024-11-26T11:15:17,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/7a832c24c08e40cca5da36c7edc331b3, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T11:15:17,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/1a73cee5cbe5478ba58dca90d1110b23 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1a73cee5cbe5478ba58dca90d1110b23 2024-11-26T11:15:17,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1a73cee5cbe5478ba58dca90d1110b23, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T11:15:17,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b258fbd035e77c2729be5c41f0c32984 in 931ms, sequenceid=156, compaction requested=false 2024-11-26T11:15:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:17,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:15:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619777459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619777459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619777462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619777462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268ea5fa435ea1439bbac5f20e68c80396_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:17,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741969_1145 (size=14794) 2024-11-26T11:15:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619777562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619777572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619777573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619777573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619777772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619777783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619777783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619777783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:17,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:17,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268ea5fa435ea1439bbac5f20e68c80396_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268ea5fa435ea1439bbac5f20e68c80396_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:17,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/546887f805294548bd206439b8f72f8b, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:17,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/546887f805294548bd206439b8f72f8b is 175, key is test_row_0/A:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:17,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741970_1146 (size=39749) 2024-11-26T11:15:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T11:15:17,933 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:17,933 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-26T11:15:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:17,935 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:17,935 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:17,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:18,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619778082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619778093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619778093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619778093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:18,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/546887f805294548bd206439b8f72f8b 2024-11-26T11:15:18,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/db9ba42ea4924b7586a73274f78d5df8 is 50, key is test_row_0/B:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:18,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741971_1147 (size=12151) 2024-11-26T11:15:18,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:18,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:18,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:18,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:18,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:18,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619778592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619778603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619778603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619778603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:18,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/db9ba42ea4924b7586a73274f78d5df8 2024-11-26T11:15:18,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8e28ee31bf7642bfbc063b0d0c6510e2 is 50, key is test_row_0/C:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741972_1148 (size=12151) 2024-11-26T11:15:18,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:18,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:18,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:19,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8e28ee31bf7642bfbc063b0d0c6510e2 2024-11-26T11:15:19,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/546887f805294548bd206439b8f72f8b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b 2024-11-26T11:15:19,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b, entries=200, sequenceid=179, filesize=38.8 K 2024-11-26T11:15:19,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/db9ba42ea4924b7586a73274f78d5df8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/db9ba42ea4924b7586a73274f78d5df8 2024-11-26T11:15:19,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/db9ba42ea4924b7586a73274f78d5df8, entries=150, sequenceid=179, filesize=11.9 K 2024-11-26T11:15:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8e28ee31bf7642bfbc063b0d0c6510e2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8e28ee31bf7642bfbc063b0d0c6510e2 2024-11-26T11:15:19,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8e28ee31bf7642bfbc063b0d0c6510e2, entries=150, sequenceid=179, filesize=11.9 K 2024-11-26T11:15:19,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b258fbd035e77c2729be5c41f0c32984 in 1744ms, sequenceid=179, compaction requested=true 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:19,190 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:19,190 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:19,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:19,191 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:19,191 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:19,191 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,192 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f11abfd03ece4ebfa1091c97be223a1d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/7a832c24c08e40cca5da36c7edc331b3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/db9ba42ea4924b7586a73274f78d5df8] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=35.8 K 2024-11-26T11:15:19,192 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:19,192 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:19,192 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f11abfd03ece4ebfa1091c97be223a1d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619715889 2024-11-26T11:15:19,192 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,192 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/f85340627413484e9144d3a941fa04f7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=108.2 K 2024-11-26T11:15:19,192 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,192 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/f85340627413484e9144d3a941fa04f7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b] 2024-11-26T11:15:19,192 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a832c24c08e40cca5da36c7edc331b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732619715939 2024-11-26T11:15:19,193 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting db9ba42ea4924b7586a73274f78d5df8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732619716291 2024-11-26T11:15:19,193 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85340627413484e9144d3a941fa04f7, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619715889 2024-11-26T11:15:19,193 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8126293f831b4be5a774ca0865ff4ca5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732619715939 2024-11-26T11:15:19,194 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546887f805294548bd206439b8f72f8b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732619716291 2024-11-26T11:15:19,204 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:19,204 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/220f033b24e14438bd4f4c37c32fbf0e is 50, key is test_row_0/B:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:19,205 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:19,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741973_1149 (size=12493) 2024-11-26T11:15:19,213 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264918dbc1d6e7436c891497ccb6524285_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:19,216 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264918dbc1d6e7436c891497ccb6524285_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:19,216 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264918dbc1d6e7436c891497ccb6524285_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:19,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741974_1150 (size=4469) 2024-11-26T11:15:19,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T11:15:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,318 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:15:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a2b2b39dc3034d569a04ae1c185fe998_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619717459/Put/seqid=0 2024-11-26T11:15:19,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741975_1151 (size=12304) 2024-11-26T11:15:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:19,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:19,626 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/220f033b24e14438bd4f4c37c32fbf0e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/220f033b24e14438bd4f4c37c32fbf0e 2024-11-26T11:15:19,627 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:19,628 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/12804de0664945ba95ea2299bb658889 is 175, key is test_row_0/A:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:19,636 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into 220f033b24e14438bd4f4c37c32fbf0e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:19,636 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:19,636 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619719190; duration=0sec 2024-11-26T11:15:19,637 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:19,637 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:19,637 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:19,640 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:19,640 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:19,640 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:19,640 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ce2e196a84c54848aa27c8b4f70714f2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1a73cee5cbe5478ba58dca90d1110b23, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8e28ee31bf7642bfbc063b0d0c6510e2] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=35.8 K 2024-11-26T11:15:19,641 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2e196a84c54848aa27c8b4f70714f2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619715889 2024-11-26T11:15:19,641 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a73cee5cbe5478ba58dca90d1110b23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732619715939 2024-11-26T11:15:19,641 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e28ee31bf7642bfbc063b0d0c6510e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732619716291 2024-11-26T11:15:19,654 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:19,655 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/728e286db084406580e3076e0e0247ee is 50, key is test_row_0/C:col10/1732619716291/Put/seqid=0 2024-11-26T11:15:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619779651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619779653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619779654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741976_1152 (size=31447) 2024-11-26T11:15:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619779656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,672 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/12804de0664945ba95ea2299bb658889 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/12804de0664945ba95ea2299bb658889 2024-11-26T11:15:19,681 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into 12804de0664945ba95ea2299bb658889(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:19,681 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:19,681 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619719190; duration=0sec 2024-11-26T11:15:19,681 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:19,681 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:19,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741977_1153 (size=12493) 2024-11-26T11:15:19,693 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/728e286db084406580e3076e0e0247ee as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/728e286db084406580e3076e0e0247ee 2024-11-26T11:15:19,712 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into 728e286db084406580e3076e0e0247ee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:19,712 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:19,712 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619719190; duration=0sec 2024-11-26T11:15:19,712 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:19,712 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:19,739 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a2b2b39dc3034d569a04ae1c185fe998_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a2b2b39dc3034d569a04ae1c185fe998_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:19,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/0e86bb9a57c44694b88c2f29ec302bb7, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/0e86bb9a57c44694b88c2f29ec302bb7 is 175, key is test_row_0/A:col10/1732619717459/Put/seqid=0 2024-11-26T11:15:19,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741978_1154 (size=31105) 2024-11-26T11:15:19,756 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/0e86bb9a57c44694b88c2f29ec302bb7 2024-11-26T11:15:19,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619779763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619779773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619779773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619779773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/fbb293b9297847e58bad2a5a7ad0a0cd is 50, key is test_row_0/B:col10/1732619717459/Put/seqid=0 2024-11-26T11:15:19,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741979_1155 (size=12151) 2024-11-26T11:15:19,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619779983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619779983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619779983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619779992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619780002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,004 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in b258fbd035e77c2729be5c41f0c32984 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:20,231 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/fbb293b9297847e58bad2a5a7ad0a0cd 2024-11-26T11:15:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/23e3e3bf5a094c1a979b36e4176157e1 is 50, key is test_row_0/C:col10/1732619717459/Put/seqid=0 2024-11-26T11:15:20,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741980_1156 (size=12151) 2024-11-26T11:15:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619780293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619780293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619780293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619780302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,646 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/23e3e3bf5a094c1a979b36e4176157e1 2024-11-26T11:15:20,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/0e86bb9a57c44694b88c2f29ec302bb7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7 2024-11-26T11:15:20,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7, entries=150, sequenceid=191, filesize=30.4 K 2024-11-26T11:15:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/fbb293b9297847e58bad2a5a7ad0a0cd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fbb293b9297847e58bad2a5a7ad0a0cd 2024-11-26T11:15:20,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fbb293b9297847e58bad2a5a7ad0a0cd, entries=150, sequenceid=191, filesize=11.9 K 2024-11-26T11:15:20,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/23e3e3bf5a094c1a979b36e4176157e1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/23e3e3bf5a094c1a979b36e4176157e1 2024-11-26T11:15:20,666 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/23e3e3bf5a094c1a979b36e4176157e1, entries=150, sequenceid=191, filesize=11.9 K 2024-11-26T11:15:20,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b258fbd035e77c2729be5c41f0c32984 in 1350ms, sequenceid=191, compaction requested=false 2024-11-26T11:15:20,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:20,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:20,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-26T11:15:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-26T11:15:20,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T11:15:20,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7340 sec 2024-11-26T11:15:20,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.7390 sec 2024-11-26T11:15:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:20,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:20,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126283ac3eae6c84fe8aaeebd00d90974eb_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:20,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741981_1157 (size=12304) 2024-11-26T11:15:20,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619780814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619780819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619780820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619780820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126283ac3eae6c84fe8aaeebd00d90974eb_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126283ac3eae6c84fe8aaeebd00d90974eb_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:20,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/fe78607ceb3248a9911e289fc98e2dbc, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:20,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/fe78607ceb3248a9911e289fc98e2dbc is 175, key is test_row_0/A:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:20,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741982_1158 (size=31105) 2024-11-26T11:15:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619780922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619780933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619780933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619780933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619781133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619781143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619781143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619781143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,238 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/fe78607ceb3248a9911e289fc98e2dbc 2024-11-26T11:15:21,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/777de404deab4a87821202600edb9842 is 50, key is test_row_0/B:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:21,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741983_1159 (size=12151) 2024-11-26T11:15:21,282 INFO [master/7f3a3edaecc3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T11:15:21,282 INFO [master/7f3a3edaecc3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T11:15:21,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619781442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619781452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619781453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619781453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/777de404deab4a87821202600edb9842 2024-11-26T11:15:21,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8158f8588f5642678563d317c8269000 is 50, key is test_row_0/C:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:21,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741984_1160 (size=12151) 2024-11-26T11:15:21,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619781952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619781963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619781963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:21,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619781973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:22,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8158f8588f5642678563d317c8269000 2024-11-26T11:15:22,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/fe78607ceb3248a9911e289fc98e2dbc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc 2024-11-26T11:15:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T11:15:22,103 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:22,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-26T11:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T11:15:22,106 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:22,106 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:22,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:22,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc, entries=150, sequenceid=215, filesize=30.4 K 2024-11-26T11:15:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/777de404deab4a87821202600edb9842 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/777de404deab4a87821202600edb9842 2024-11-26T11:15:22,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/777de404deab4a87821202600edb9842, entries=150, sequenceid=215, filesize=11.9 K 2024-11-26T11:15:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8158f8588f5642678563d317c8269000 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8158f8588f5642678563d317c8269000 2024-11-26T11:15:22,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8158f8588f5642678563d317c8269000, entries=150, sequenceid=215, filesize=11.9 K 2024-11-26T11:15:22,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for b258fbd035e77c2729be5c41f0c32984 in 1318ms, sequenceid=215, compaction requested=true 2024-11-26T11:15:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:22,124 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:22,124 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:22,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:22,125 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:22,125 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:22,125 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:22,125 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:22,126 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:22,126 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/220f033b24e14438bd4f4c37c32fbf0e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fbb293b9297847e58bad2a5a7ad0a0cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/777de404deab4a87821202600edb9842] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=35.9 K 2024-11-26T11:15:22,126 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:22,126 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/12804de0664945ba95ea2299bb658889, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=91.5 K 2024-11-26T11:15:22,126 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:22,126 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/12804de0664945ba95ea2299bb658889, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc] 2024-11-26T11:15:22,126 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 220f033b24e14438bd4f4c37c32fbf0e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732619716291 2024-11-26T11:15:22,127 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12804de0664945ba95ea2299bb658889, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732619716291 2024-11-26T11:15:22,128 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb293b9297847e58bad2a5a7ad0a0cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732619717457 2024-11-26T11:15:22,129 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e86bb9a57c44694b88c2f29ec302bb7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732619717457 2024-11-26T11:15:22,129 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 777de404deab4a87821202600edb9842, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732619719625 2024-11-26T11:15:22,129 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe78607ceb3248a9911e289fc98e2dbc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732619719625 2024-11-26T11:15:22,136 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:22,138 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:22,138 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/4246c1f3daf447adb96ab7f30a145020 is 50, key is test_row_0/B:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:22,144 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126096bb6daabf24ddaa082ea6da9e97a99_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:22,146 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126096bb6daabf24ddaa082ea6da9e97a99_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:22,146 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126096bb6daabf24ddaa082ea6da9e97a99_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741986_1162 (size=4469) 2024-11-26T11:15:22,157 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#133 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741985_1161 (size=12595) 2024-11-26T11:15:22,157 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/0d26e5c88dce491a882fc9d45408455a is 175, key is test_row_0/A:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:22,167 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/4246c1f3daf447adb96ab7f30a145020 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/4246c1f3daf447adb96ab7f30a145020 2024-11-26T11:15:22,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741987_1163 (size=31549) 2024-11-26T11:15:22,178 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into 4246c1f3daf447adb96ab7f30a145020(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:22,178 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:22,178 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619722124; duration=0sec 2024-11-26T11:15:22,178 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:22,178 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:22,178 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:22,181 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:22,181 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:22,181 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:22,181 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/728e286db084406580e3076e0e0247ee, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/23e3e3bf5a094c1a979b36e4176157e1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8158f8588f5642678563d317c8269000] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=35.9 K 2024-11-26T11:15:22,182 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 728e286db084406580e3076e0e0247ee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732619716291 2024-11-26T11:15:22,182 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e3e3bf5a094c1a979b36e4176157e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732619717457 2024-11-26T11:15:22,183 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 8158f8588f5642678563d317c8269000, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732619719625 2024-11-26T11:15:22,191 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:22,192 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/46f4c760855c441d982633af67474054 is 50, key is test_row_0/C:col10/1732619720803/Put/seqid=0 2024-11-26T11:15:22,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741988_1164 (size=12595) 2024-11-26T11:15:22,208 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/46f4c760855c441d982633af67474054 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46f4c760855c441d982633af67474054 2024-11-26T11:15:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T11:15:22,218 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into 46f4c760855c441d982633af67474054(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:22,218 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:22,218 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619722124; duration=0sec 2024-11-26T11:15:22,218 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:22,218 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:22,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T11:15:22,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:22,260 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265894c3d815a74907937ea4fb078a2599_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619720819/Put/seqid=0 2024-11-26T11:15:22,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741989_1165 (size=12304) 2024-11-26T11:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T11:15:22,581 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/0d26e5c88dce491a882fc9d45408455a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0d26e5c88dce491a882fc9d45408455a 2024-11-26T11:15:22,586 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into 0d26e5c88dce491a882fc9d45408455a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:22,586 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:22,586 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619722124; duration=0sec 2024-11-26T11:15:22,586 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:22,586 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:22,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265894c3d815a74907937ea4fb078a2599_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265894c3d815a74907937ea4fb078a2599_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:22,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/7b6a274c86934b03bd77a23729d9c5a8, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:22,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/7b6a274c86934b03bd77a23729d9c5a8 is 175, key is test_row_0/A:col10/1732619720819/Put/seqid=0 2024-11-26T11:15:22,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741990_1166 (size=31105) 2024-11-26T11:15:22,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/7b6a274c86934b03bd77a23729d9c5a8 2024-11-26T11:15:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/8502294202d4429aa86e445c78b9b601 is 50, key is test_row_0/B:col10/1732619720819/Put/seqid=0 2024-11-26T11:15:22,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741991_1167 (size=12151) 2024-11-26T11:15:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T11:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619783018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619783020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619783021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619783022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,125 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/8502294202d4429aa86e445c78b9b601 2024-11-26T11:15:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/e52d2795e3f24d03a1f64cefc85d2ece is 50, key is test_row_0/C:col10/1732619720819/Put/seqid=0 2024-11-26T11:15:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619783133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619783133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619783133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619783133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741992_1168 (size=12151) 2024-11-26T11:15:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T11:15:23,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619783343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619783344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619783345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619783345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,542 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/e52d2795e3f24d03a1f64cefc85d2ece 2024-11-26T11:15:23,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/7b6a274c86934b03bd77a23729d9c5a8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8 2024-11-26T11:15:23,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8, entries=150, sequenceid=229, filesize=30.4 K 2024-11-26T11:15:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/8502294202d4429aa86e445c78b9b601 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/8502294202d4429aa86e445c78b9b601 2024-11-26T11:15:23,556 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/8502294202d4429aa86e445c78b9b601, entries=150, sequenceid=229, filesize=11.9 K 2024-11-26T11:15:23,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/e52d2795e3f24d03a1f64cefc85d2ece as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/e52d2795e3f24d03a1f64cefc85d2ece 2024-11-26T11:15:23,563 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/e52d2795e3f24d03a1f64cefc85d2ece, entries=150, sequenceid=229, filesize=11.9 K 2024-11-26T11:15:23,564 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b258fbd035e77c2729be5c41f0c32984 in 1305ms, sequenceid=229, compaction requested=false 2024-11-26T11:15:23,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:23,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:23,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-26T11:15:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-26T11:15:23,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-26T11:15:23,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-26T11:15:23,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.4670 sec 2024-11-26T11:15:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:23,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T11:15:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:23,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c4bc3fc717fd4d1db655169fb256283d_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:23,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619783675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619783679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619783679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741993_1169 (size=12304) 2024-11-26T11:15:23,688 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:23,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c4bc3fc717fd4d1db655169fb256283d_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c4bc3fc717fd4d1db655169fb256283d_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:23,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619783681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,697 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9aa9cb4338d140989f413da10358f06d, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:23,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9aa9cb4338d140989f413da10358f06d is 175, key is test_row_0/A:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741994_1170 (size=31105) 2024-11-26T11:15:23,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9aa9cb4338d140989f413da10358f06d 2024-11-26T11:15:23,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/0b39e534742f44c189072ddbc6f87a96 is 50, key is test_row_0/B:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:23,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741995_1171 (size=12151) 2024-11-26T11:15:23,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619783783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619783793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619783793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619783803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:23,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619783993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619784003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619784003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619784012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/0b39e534742f44c189072ddbc6f87a96 2024-11-26T11:15:24,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/1119fe710f6345178decf1cfe190f9f0 is 50, key is test_row_0/C:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:24,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741996_1172 (size=12151) 2024-11-26T11:15:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T11:15:24,253 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:24,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-26T11:15:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T11:15:24,255 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:24,256 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:24,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619784302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619784313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619784313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619784323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T11:15:24,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T11:15:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:24,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/1119fe710f6345178decf1cfe190f9f0 2024-11-26T11:15:24,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9aa9cb4338d140989f413da10358f06d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d 2024-11-26T11:15:24,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d, entries=150, sequenceid=251, filesize=30.4 K 2024-11-26T11:15:24,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/0b39e534742f44c189072ddbc6f87a96 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/0b39e534742f44c189072ddbc6f87a96 2024-11-26T11:15:24,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T11:15:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:24,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:24,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/0b39e534742f44c189072ddbc6f87a96, entries=150, sequenceid=251, filesize=11.9 K 2024-11-26T11:15:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:24,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/1119fe710f6345178decf1cfe190f9f0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1119fe710f6345178decf1cfe190f9f0 2024-11-26T11:15:24,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1119fe710f6345178decf1cfe190f9f0, entries=150, sequenceid=251, filesize=11.9 K 2024-11-26T11:15:24,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for b258fbd035e77c2729be5c41f0c32984 in 913ms, sequenceid=251, compaction requested=true 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:24,569 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:24,569 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:24,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:24,570 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:24,570 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:24,570 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:24,570 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:24,570 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,570 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,570 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/4246c1f3daf447adb96ab7f30a145020, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/8502294202d4429aa86e445c78b9b601, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/0b39e534742f44c189072ddbc6f87a96] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.0 K 2024-11-26T11:15:24,570 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0d26e5c88dce491a882fc9d45408455a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=91.6 K 2024-11-26T11:15:24,570 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,570 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0d26e5c88dce491a882fc9d45408455a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d] 2024-11-26T11:15:24,571 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4246c1f3daf447adb96ab7f30a145020, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732619719625 2024-11-26T11:15:24,571 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d26e5c88dce491a882fc9d45408455a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732619719625 2024-11-26T11:15:24,571 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 8502294202d4429aa86e445c78b9b601, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732619720805 2024-11-26T11:15:24,571 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6a274c86934b03bd77a23729d9c5a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732619720805 2024-11-26T11:15:24,571 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b39e534742f44c189072ddbc6f87a96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732619722985 2024-11-26T11:15:24,571 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aa9cb4338d140989f413da10358f06d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732619722985 2024-11-26T11:15:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T11:15:24,577 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:24,579 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268af7523f5fd94b129e9f94b8835cda6a_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:24,580 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:24,581 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/c6a38bf4d1fc4bd2a6fd72aca1048954 is 50, key is test_row_0/B:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:24,581 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268af7523f5fd94b129e9f94b8835cda6a_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:24,581 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268af7523f5fd94b129e9f94b8835cda6a_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741997_1173 (size=12697) 2024-11-26T11:15:24,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741998_1174 (size=4469) 2024-11-26T11:15:24,593 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#141 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:24,594 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/720308c32bdd4c9bbf88c714e9601c98 is 175, key is test_row_0/A:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741999_1175 (size=31651) 2024-11-26T11:15:24,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T11:15:24,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:24,714 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:15:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:24,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112613cd655def224a768bbb6bd15f80b2f0_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619723670/Put/seqid=0 2024-11-26T11:15:24,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742000_1176 (size=12454) 2024-11-26T11:15:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:24,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619784856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619784856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619784856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619784857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T11:15:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619784963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619784963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619784963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619784963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:24,997 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/c6a38bf4d1fc4bd2a6fd72aca1048954 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c6a38bf4d1fc4bd2a6fd72aca1048954 2024-11-26T11:15:25,002 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/720308c32bdd4c9bbf88c714e9601c98 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/720308c32bdd4c9bbf88c714e9601c98 2024-11-26T11:15:25,002 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into c6a38bf4d1fc4bd2a6fd72aca1048954(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:25,002 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:25,002 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619724569; duration=0sec 2024-11-26T11:15:25,003 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:25,003 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:25,003 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:25,004 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:25,004 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:25,004 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:25,004 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46f4c760855c441d982633af67474054, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/e52d2795e3f24d03a1f64cefc85d2ece, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1119fe710f6345178decf1cfe190f9f0] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.0 K 2024-11-26T11:15:25,005 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f4c760855c441d982633af67474054, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732619719625 2024-11-26T11:15:25,006 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e52d2795e3f24d03a1f64cefc85d2ece, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732619720805 2024-11-26T11:15:25,007 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1119fe710f6345178decf1cfe190f9f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732619722985 2024-11-26T11:15:25,008 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into 720308c32bdd4c9bbf88c714e9601c98(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:25,008 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:25,008 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619724569; duration=0sec 2024-11-26T11:15:25,008 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:25,008 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:25,014 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:25,014 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/bf75544f16bd47f1af807c98919169bc is 50, key is test_row_0/C:col10/1732619722985/Put/seqid=0 2024-11-26T11:15:25,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742001_1177 (size=12697) 2024-11-26T11:15:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:25,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112613cd655def224a768bbb6bd15f80b2f0_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112613cd655def224a768bbb6bd15f80b2f0_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:25,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/a33f7129f0bc4acebb0c66fc1b7087c2, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:25,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/a33f7129f0bc4acebb0c66fc1b7087c2 is 175, key is test_row_0/A:col10/1732619723670/Put/seqid=0 2024-11-26T11:15:25,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742002_1178 (size=31255) 2024-11-26T11:15:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619785173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619785173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619785173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619785173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T11:15:25,424 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/bf75544f16bd47f1af807c98919169bc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bf75544f16bd47f1af807c98919169bc 2024-11-26T11:15:25,429 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into bf75544f16bd47f1af807c98919169bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:25,429 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:25,429 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619724569; duration=0sec 2024-11-26T11:15:25,429 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:25,429 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619785482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619785483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619785483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619785483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:25,535 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/a33f7129f0bc4acebb0c66fc1b7087c2 2024-11-26T11:15:25,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/307b4828c2bb474c96f5c2000d246207 is 50, key is test_row_0/B:col10/1732619723670/Put/seqid=0 2024-11-26T11:15:25,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742003_1179 (size=12301) 2024-11-26T11:15:25,548 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/307b4828c2bb474c96f5c2000d246207 2024-11-26T11:15:25,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/bd631c7e9ab942569b4ce70fe61153fb is 50, key is test_row_0/C:col10/1732619723670/Put/seqid=0 2024-11-26T11:15:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742004_1180 (size=12301) 2024-11-26T11:15:25,561 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/bd631c7e9ab942569b4ce70fe61153fb 2024-11-26T11:15:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/a33f7129f0bc4acebb0c66fc1b7087c2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2 2024-11-26T11:15:25,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2, entries=150, sequenceid=263, filesize=30.5 K 2024-11-26T11:15:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/307b4828c2bb474c96f5c2000d246207 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/307b4828c2bb474c96f5c2000d246207 2024-11-26T11:15:25,575 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/307b4828c2bb474c96f5c2000d246207, entries=150, sequenceid=263, filesize=12.0 K 2024-11-26T11:15:25,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/bd631c7e9ab942569b4ce70fe61153fb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bd631c7e9ab942569b4ce70fe61153fb 2024-11-26T11:15:25,581 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bd631c7e9ab942569b4ce70fe61153fb, entries=150, sequenceid=263, filesize=12.0 K 2024-11-26T11:15:25,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b258fbd035e77c2729be5c41f0c32984 in 868ms, sequenceid=263, compaction requested=false 2024-11-26T11:15:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-26T11:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-26T11:15:25,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T11:15:25,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-26T11:15:25,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.3320 sec 2024-11-26T11:15:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:25,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T11:15:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:26,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b6f86f093b6b44bb8a3760c268a2a66b_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:26,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742005_1181 (size=14994) 2024-11-26T11:15:26,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:26,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b6f86f093b6b44bb8a3760c268a2a66b_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b6f86f093b6b44bb8a3760c268a2a66b_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:26,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/00e2479d5e3847e2ab449d21d10c12ca, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:26,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/00e2479d5e3847e2ab449d21d10c12ca is 175, key is test_row_0/A:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619786007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742006_1182 (size=39949) 2024-11-26T11:15:26,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619786012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619786014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619786014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619786123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619786123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619786123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619786123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619786333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619786333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619786333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619786333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T11:15:26,413 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:26,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-26T11:15:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T11:15:26,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:26,416 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/00e2479d5e3847e2ab449d21d10c12ca 2024-11-26T11:15:26,416 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:26,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:26,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/a31b41965e01408f828cdd82b5157308 is 50, key is test_row_0/B:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:26,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742007_1183 (size=12301) 2024-11-26T11:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T11:15:26,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T11:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:26,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619786643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619786643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619786643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619786643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:26,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T11:15:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:26,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T11:15:26,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/a31b41965e01408f828cdd82b5157308 2024-11-26T11:15:26,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/20b20c7ff3984a9f98b475465a665385 is 50, key is test_row_0/C:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742008_1184 (size=12301) 2024-11-26T11:15:26,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T11:15:26,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:26,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:26,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:26,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T11:15:27,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:27,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T11:15:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619787152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619787152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619787152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619787163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:27,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T11:15:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:27,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/20b20c7ff3984a9f98b475465a665385 2024-11-26T11:15:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/00e2479d5e3847e2ab449d21d10c12ca as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca 2024-11-26T11:15:27,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca, entries=200, sequenceid=287, filesize=39.0 K 2024-11-26T11:15:27,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/a31b41965e01408f828cdd82b5157308 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a31b41965e01408f828cdd82b5157308 2024-11-26T11:15:27,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a31b41965e01408f828cdd82b5157308, entries=150, sequenceid=287, filesize=12.0 K 2024-11-26T11:15:27,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/20b20c7ff3984a9f98b475465a665385 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/20b20c7ff3984a9f98b475465a665385 2024-11-26T11:15:27,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/20b20c7ff3984a9f98b475465a665385, entries=150, sequenceid=287, filesize=12.0 K 2024-11-26T11:15:27,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for b258fbd035e77c2729be5c41f0c32984 in 1273ms, sequenceid=287, compaction requested=true 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:27,267 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:27,267 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:27,268 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:27,268 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:27,268 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:27,268 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:27,269 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,269 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,269 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/720308c32bdd4c9bbf88c714e9601c98, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=100.4 K 2024-11-26T11:15:27,269 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c6a38bf4d1fc4bd2a6fd72aca1048954, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/307b4828c2bb474c96f5c2000d246207, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a31b41965e01408f828cdd82b5157308] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.4 K 2024-11-26T11:15:27,269 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,269 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/720308c32bdd4c9bbf88c714e9601c98, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca] 2024-11-26T11:15:27,269 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720308c32bdd4c9bbf88c714e9601c98, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732619722985 2024-11-26T11:15:27,269 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a38bf4d1fc4bd2a6fd72aca1048954, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732619722985 2024-11-26T11:15:27,270 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 307b4828c2bb474c96f5c2000d246207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732619723670 2024-11-26T11:15:27,270 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33f7129f0bc4acebb0c66fc1b7087c2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732619723670 2024-11-26T11:15:27,270 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a31b41965e01408f828cdd82b5157308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732619724852 2024-11-26T11:15:27,270 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e2479d5e3847e2ab449d21d10c12ca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732619724852 2024-11-26T11:15:27,279 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:27,280 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/56f6d9ecbe0e4038a2ae14a410d147e4 is 50, key is test_row_0/B:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:27,281 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:27,293 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411266b315b37ce084c8283dafead92d02d27_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:27,294 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411266b315b37ce084c8283dafead92d02d27_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:27,294 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266b315b37ce084c8283dafead92d02d27_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:27,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742009_1185 (size=12949) 2024-11-26T11:15:27,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742010_1186 (size=4469) 2024-11-26T11:15:27,317 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#151 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:27,317 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/e78aceccba654f10bf99653c84fd98d6 is 175, key is test_row_0/A:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:27,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742011_1187 (size=31903) 2024-11-26T11:15:27,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,334 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fa4a7fd6d70c4db29f65a91ba8254944_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619726011/Put/seqid=0 2024-11-26T11:15:27,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742012_1188 (size=12454) 2024-11-26T11:15:27,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,355 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fa4a7fd6d70c4db29f65a91ba8254944_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fa4a7fd6d70c4db29f65a91ba8254944_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:27,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c5ba2b6955da417a9c20dfe48f8a5261, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:27,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c5ba2b6955da417a9c20dfe48f8a5261 is 175, key is test_row_0/A:col10/1732619726011/Put/seqid=0 2024-11-26T11:15:27,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742013_1189 (size=31255) 2024-11-26T11:15:27,362 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c5ba2b6955da417a9c20dfe48f8a5261 2024-11-26T11:15:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/c8ee46ee692a40bf8e4b84e18b622365 is 50, key is test_row_0/B:col10/1732619726011/Put/seqid=0 2024-11-26T11:15:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742014_1190 (size=12301) 2024-11-26T11:15:27,385 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/c8ee46ee692a40bf8e4b84e18b622365 2024-11-26T11:15:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/f2e0f08a0b304ffdb15a0856757ab8ff is 50, key is test_row_0/C:col10/1732619726011/Put/seqid=0 2024-11-26T11:15:27,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742015_1191 (size=12301) 2024-11-26T11:15:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T11:15:27,706 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/56f6d9ecbe0e4038a2ae14a410d147e4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/56f6d9ecbe0e4038a2ae14a410d147e4 2024-11-26T11:15:27,720 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into 56f6d9ecbe0e4038a2ae14a410d147e4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:27,720 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:27,720 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619727267; duration=0sec 2024-11-26T11:15:27,720 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:27,720 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:27,721 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:27,722 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:27,722 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:27,722 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,723 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bf75544f16bd47f1af807c98919169bc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bd631c7e9ab942569b4ce70fe61153fb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/20b20c7ff3984a9f98b475465a665385] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.4 K 2024-11-26T11:15:27,724 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting bf75544f16bd47f1af807c98919169bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732619722985 2024-11-26T11:15:27,724 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting bd631c7e9ab942569b4ce70fe61153fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732619723670 2024-11-26T11:15:27,726 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b20c7ff3984a9f98b475465a665385, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732619724852 2024-11-26T11:15:27,727 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/e78aceccba654f10bf99653c84fd98d6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/e78aceccba654f10bf99653c84fd98d6 2024-11-26T11:15:27,733 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into e78aceccba654f10bf99653c84fd98d6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:27,733 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:27,733 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619727267; duration=0sec 2024-11-26T11:15:27,733 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:27,733 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:27,736 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:27,737 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/3ab835bf552649e680804ca68e8ee75a is 50, key is test_row_0/C:col10/1732619724855/Put/seqid=0 2024-11-26T11:15:27,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742016_1192 (size=12949) 2024-11-26T11:15:27,802 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/f2e0f08a0b304ffdb15a0856757ab8ff 2024-11-26T11:15:27,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/c5ba2b6955da417a9c20dfe48f8a5261 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261 2024-11-26T11:15:27,813 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261, entries=150, sequenceid=299, filesize=30.5 K 2024-11-26T11:15:27,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/c8ee46ee692a40bf8e4b84e18b622365 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c8ee46ee692a40bf8e4b84e18b622365 2024-11-26T11:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,822 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c8ee46ee692a40bf8e4b84e18b622365, entries=150, sequenceid=299, filesize=12.0 K 2024-11-26T11:15:27,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/f2e0f08a0b304ffdb15a0856757ab8ff as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/f2e0f08a0b304ffdb15a0856757ab8ff 2024-11-26T11:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,828 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/f2e0f08a0b304ffdb15a0856757ab8ff, entries=150, sequenceid=299, filesize=12.0 K 2024-11-26T11:15:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,829 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for b258fbd035e77c2729be5c41f0c32984 in 495ms, sequenceid=299, compaction requested=false 2024-11-26T11:15:27,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:27,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:27,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-26T11:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-26T11:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T11:15:27,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-26T11:15:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4200 sec 2024-11-26T11:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,157 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/3ab835bf552649e680804ca68e8ee75a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3ab835bf552649e680804ca68e8ee75a 2024-11-26T11:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,163 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into 3ab835bf552649e680804ca68e8ee75a(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:28,163 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:28,163 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619727267; duration=0sec 2024-11-26T11:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,164 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,164 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:28,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e4dfd78c3f5841b6967ba17e510b134d_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742017_1193 (size=22618) 2024-11-26T11:15:28,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732619728281, max timestamp=1732619728293Num uniques 859; 2024-11-26T11:15:28,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732619728281, max timestamp=1732619728293Num uniques 859; 2024-11-26T11:15:28,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T11:15:28,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T11:15:28,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732619728281, max timestamp=1732619728293Num uniques 859; 2024-11-26T11:15:28,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T11:15:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619788323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619788327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619788329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619788329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619788433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619788433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619788443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619788443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T11:15:28,563 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:28,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-26T11:15:28,566 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:28,566 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:28,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619788643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619788643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619788653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619788653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:28,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:28,682 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e4dfd78c3f5841b6967ba17e510b134d_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e4dfd78c3f5841b6967ba17e510b134d_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:28,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/ef3da29e865c4431ae9f82e773e0fe44, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:28,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/ef3da29e865c4431ae9f82e773e0fe44 is 175, key is test_row_0/A:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:28,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742018_1194 (size=66023) 2024-11-26T11:15:28,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/ef3da29e865c4431ae9f82e773e0fe44 2024-11-26T11:15:28,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/17169272ae494124ab8e59ae2f9ffa89 is 50, key is test_row_0/B:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:28,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742019_1195 (size=12301) 2024-11-26T11:15:28,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T11:15:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:28,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:28,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/17169272ae494124ab8e59ae2f9ffa89 2024-11-26T11:15:28,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/3c8b628efa0143f2835685ee00367cea is 50, key is test_row_0/C:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742020_1196 (size=12301) 2024-11-26T11:15:28,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T11:15:28,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:28,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:28,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:28,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:28,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619788952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619788952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619788963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619788963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T11:15:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:29,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/3c8b628efa0143f2835685ee00367cea 2024-11-26T11:15:29,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/ef3da29e865c4431ae9f82e773e0fe44 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44 2024-11-26T11:15:29,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44, entries=350, sequenceid=312, filesize=64.5 K 2024-11-26T11:15:29,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/17169272ae494124ab8e59ae2f9ffa89 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/17169272ae494124ab8e59ae2f9ffa89 2024-11-26T11:15:29,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/17169272ae494124ab8e59ae2f9ffa89, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T11:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/3c8b628efa0143f2835685ee00367cea as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3c8b628efa0143f2835685ee00367cea 2024-11-26T11:15:29,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3c8b628efa0143f2835685ee00367cea, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T11:15:29,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b258fbd035e77c2729be5c41f0c32984 in 913ms, sequenceid=312, compaction requested=true 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:29,161 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:29,161 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:29,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:29,162 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:29,162 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:29,162 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:29,162 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:29,162 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,162 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,162 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/56f6d9ecbe0e4038a2ae14a410d147e4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c8ee46ee692a40bf8e4b84e18b622365, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/17169272ae494124ab8e59ae2f9ffa89] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.7 K 2024-11-26T11:15:29,162 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/e78aceccba654f10bf99653c84fd98d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=126.2 K 2024-11-26T11:15:29,163 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,163 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/e78aceccba654f10bf99653c84fd98d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44] 2024-11-26T11:15:29,163 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f6d9ecbe0e4038a2ae14a410d147e4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732619724852 2024-11-26T11:15:29,163 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ee46ee692a40bf8e4b84e18b622365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732619725994 2024-11-26T11:15:29,163 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e78aceccba654f10bf99653c84fd98d6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732619724852 2024-11-26T11:15:29,163 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 17169272ae494124ab8e59ae2f9ffa89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619728243 2024-11-26T11:15:29,164 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ba2b6955da417a9c20dfe48f8a5261, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732619725994 2024-11-26T11:15:29,164 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3da29e865c4431ae9f82e773e0fe44, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619728191 2024-11-26T11:15:29,169 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:29,171 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:29,171 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268f66e25073e9430bac24bde8386c9ace_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:29,171 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/6c56b2a4befc4ce6b42ac7798dfb6c68 is 50, key is test_row_0/B:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:29,173 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268f66e25073e9430bac24bde8386c9ace_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:29,173 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268f66e25073e9430bac24bde8386c9ace_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:29,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742022_1198 (size=4469) 2024-11-26T11:15:29,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T11:15:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,178 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-26T11:15:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:29,179 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#160 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:29,179 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/30c9a7c5412c4621b3ca42fcdcef5056 is 175, key is test_row_0/A:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:29,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742021_1197 (size=13051) 2024-11-26T11:15:29,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742023_1199 (size=32005) 2024-11-26T11:15:29,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112627b40270bd4040d89f3100eae325f357_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619728321/Put/seqid=0 2024-11-26T11:15:29,191 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/30c9a7c5412c4621b3ca42fcdcef5056 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/30c9a7c5412c4621b3ca42fcdcef5056 2024-11-26T11:15:29,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742024_1200 (size=12454) 2024-11-26T11:15:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:29,199 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into 30c9a7c5412c4621b3ca42fcdcef5056(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:29,199 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:29,199 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619729161; duration=0sec 2024-11-26T11:15:29,199 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:29,199 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:29,199 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:29,201 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:29,201 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:29,201 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:29,201 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3ab835bf552649e680804ca68e8ee75a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/f2e0f08a0b304ffdb15a0856757ab8ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3c8b628efa0143f2835685ee00367cea] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.7 K 2024-11-26T11:15:29,201 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab835bf552649e680804ca68e8ee75a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732619724852 2024-11-26T11:15:29,202 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2e0f08a0b304ffdb15a0856757ab8ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732619725994 2024-11-26T11:15:29,202 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8b628efa0143f2835685ee00367cea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619728243 2024-11-26T11:15:29,208 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:29,209 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/217cb3ee59bf490ba9b81fd011b87585 is 50, key is test_row_0/C:col10/1732619728246/Put/seqid=0 2024-11-26T11:15:29,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742025_1201 (size=13051) 2024-11-26T11:15:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:29,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. as already flushing 2024-11-26T11:15:29,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619789492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619789492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619789493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619789494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/6c56b2a4befc4ce6b42ac7798dfb6c68 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/6c56b2a4befc4ce6b42ac7798dfb6c68 2024-11-26T11:15:29,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:29,599 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into 6c56b2a4befc4ce6b42ac7798dfb6c68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:29,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:29,599 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619729161; duration=0sec 2024-11-26T11:15:29,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:29,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:29,601 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112627b40270bd4040d89f3100eae325f357_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112627b40270bd4040d89f3100eae325f357_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:29,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/d5b00eba17cf4ef3a4777ffe78d2b9ec, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/d5b00eba17cf4ef3a4777ffe78d2b9ec is 175, key is test_row_0/A:col10/1732619728321/Put/seqid=0 2024-11-26T11:15:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619789603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619789603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742026_1202 (size=31255) 2024-11-26T11:15:29,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619789613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619789613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,629 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/217cb3ee59bf490ba9b81fd011b87585 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/217cb3ee59bf490ba9b81fd011b87585 2024-11-26T11:15:29,635 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into 217cb3ee59bf490ba9b81fd011b87585(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:29,635 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:29,635 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619729161; duration=0sec 2024-11-26T11:15:29,635 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:29,635 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619789813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619789813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619789832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619789832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/d5b00eba17cf4ef3a4777ffe78d2b9ec 2024-11-26T11:15:30,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/f65d8f3b6454490abd4231188aa3c458 is 50, key is test_row_0/B:col10/1732619728321/Put/seqid=0 2024-11-26T11:15:30,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742027_1203 (size=12301) 2024-11-26T11:15:30,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41226 deadline: 1732619790092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,094 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in b258fbd035e77c2729be5c41f0c32984 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18350 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619790123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619790123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619790143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619790143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,422 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/f65d8f3b6454490abd4231188aa3c458 2024-11-26T11:15:30,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/4ed6f2f7b7614b59bba1e49a61ebb3f7 is 50, key is test_row_0/C:col10/1732619728321/Put/seqid=0 2024-11-26T11:15:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742028_1204 (size=12301) 2024-11-26T11:15:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41240 deadline: 1732619790633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41254 deadline: 1732619790633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41216 deadline: 1732619790652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41190 deadline: 1732619790652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:15:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:30,834 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/4ed6f2f7b7614b59bba1e49a61ebb3f7 2024-11-26T11:15:30,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/d5b00eba17cf4ef3a4777ffe78d2b9ec as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec 2024-11-26T11:15:30,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec, entries=150, sequenceid=347, filesize=30.5 K 2024-11-26T11:15:30,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/f65d8f3b6454490abd4231188aa3c458 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f65d8f3b6454490abd4231188aa3c458 2024-11-26T11:15:30,851 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f65d8f3b6454490abd4231188aa3c458, entries=150, sequenceid=347, filesize=12.0 K 2024-11-26T11:15:30,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/4ed6f2f7b7614b59bba1e49a61ebb3f7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4ed6f2f7b7614b59bba1e49a61ebb3f7 2024-11-26T11:15:30,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4ed6f2f7b7614b59bba1e49a61ebb3f7, entries=150, sequenceid=347, filesize=12.0 K 2024-11-26T11:15:30,856 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for b258fbd035e77c2729be5c41f0c32984 in 1678ms, sequenceid=347, compaction requested=false 2024-11-26T11:15:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-26T11:15:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-26T11:15:30,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T11:15:30,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2910 sec 2024-11-26T11:15:30,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2950 sec 2024-11-26T11:15:31,633 INFO [Thread-575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-575. 2024-11-26T11:15:31,633 DEBUG [Thread-575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,633 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,634 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,635 INFO [Thread-573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-573. 2024-11-26T11:15:31,636 DEBUG [Thread-573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,636 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,636 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,637 INFO [Thread-577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-577. 2024-11-26T11:15:31,637 DEBUG [Thread-577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,637 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,637 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,637 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-11-26T11:15:31,637 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,637 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,637 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:31,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T11:15:31,644 INFO [Thread-564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-564. 2024-11-26T11:15:31,644 DEBUG [Thread-564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,644 DEBUG [Thread-564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:31,644 DEBUG [Thread-564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:31,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:31,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:31,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126189a5cfe2ef04968a7e59f09df4f04c7_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:31,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742029_1205 (size=12454) 2024-11-26T11:15:31,654 INFO [Thread-566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-566. 2024-11-26T11:15:31,654 DEBUG [Thread-566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,654 DEBUG [Thread-566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,654 DEBUG [Thread-566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,675 INFO [Thread-562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-562. 2024-11-26T11:15:31,675 DEBUG [Thread-562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,675 DEBUG [Thread-562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,675 INFO [Thread-570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-570. 2024-11-26T11:15:31,675 DEBUG [Thread-562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,675 DEBUG [Thread-570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:31,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:31,675 DEBUG [Thread-570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,675 DEBUG [Thread-570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:31,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:32,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:32,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126189a5cfe2ef04968a7e59f09df4f04c7_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126189a5cfe2ef04968a7e59f09df4f04c7_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:32,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9c018ad28115420faed913ffd34d711e, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:32,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9c018ad28115420faed913ffd34d711e is 175, key is test_row_0/A:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:32,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742030_1206 (size=31255) 2024-11-26T11:15:32,472 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9c018ad28115420faed913ffd34d711e 2024-11-26T11:15:32,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/fe38244635ee4e8fbecf614d31c1db91 is 50, key is test_row_0/B:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:32,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742031_1207 (size=12301) 2024-11-26T11:15:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T11:15:32,734 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:32,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/fe38244635ee4e8fbecf614d31c1db91 2024-11-26T11:15:32,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/c91b2e53fa754b07987ab56ba82f20df is 50, key is test_row_0/C:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:32,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742032_1208 (size=12301) 2024-11-26T11:15:33,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/c91b2e53fa754b07987ab56ba82f20df 2024-11-26T11:15:33,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/9c018ad28115420faed913ffd34d711e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e 2024-11-26T11:15:33,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e, entries=150, sequenceid=361, filesize=30.5 K 2024-11-26T11:15:33,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/fe38244635ee4e8fbecf614d31c1db91 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fe38244635ee4e8fbecf614d31c1db91 2024-11-26T11:15:33,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fe38244635ee4e8fbecf614d31c1db91, entries=150, sequenceid=361, filesize=12.0 K 2024-11-26T11:15:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/c91b2e53fa754b07987ab56ba82f20df as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/c91b2e53fa754b07987ab56ba82f20df 2024-11-26T11:15:33,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/c91b2e53fa754b07987ab56ba82f20df, entries=150, sequenceid=361, filesize=12.0 K 2024-11-26T11:15:33,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for b258fbd035e77c2729be5c41f0c32984 in 1689ms, sequenceid=361, compaction requested=true 2024-11-26T11:15:33,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:33,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:33,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:33,334 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:33,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:33,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:33,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b258fbd035e77c2729be5c41f0c32984:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:33,334 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:33,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/B is initiating minor compaction (all files) 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/A is initiating minor compaction (all files) 2024-11-26T11:15:33,335 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/B in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:33,335 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/A in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:33,335 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/6c56b2a4befc4ce6b42ac7798dfb6c68, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f65d8f3b6454490abd4231188aa3c458, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fe38244635ee4e8fbecf614d31c1db91] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.8 K 2024-11-26T11:15:33,335 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/30c9a7c5412c4621b3ca42fcdcef5056, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=92.3 K 2024-11-26T11:15:33,335 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/30c9a7c5412c4621b3ca42fcdcef5056, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e] 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c56b2a4befc4ce6b42ac7798dfb6c68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619728243 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30c9a7c5412c4621b3ca42fcdcef5056, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619728243 2024-11-26T11:15:33,335 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f65d8f3b6454490abd4231188aa3c458, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732619728321 2024-11-26T11:15:33,336 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b00eba17cf4ef3a4777ffe78d2b9ec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732619728321 2024-11-26T11:15:33,336 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting fe38244635ee4e8fbecf614d31c1db91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732619729491 2024-11-26T11:15:33,336 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c018ad28115420faed913ffd34d711e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732619729491 2024-11-26T11:15:33,343 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:33,343 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/a7c6cf86c7d94c2bad34666f8e27fbb0 is 50, key is test_row_0/B:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:33,343 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:33,345 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126411c4872cdb34d0eb59d25128c980895_b258fbd035e77c2729be5c41f0c32984 store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:33,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742033_1209 (size=13153) 2024-11-26T11:15:33,361 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126411c4872cdb34d0eb59d25128c980895_b258fbd035e77c2729be5c41f0c32984, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:33,361 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126411c4872cdb34d0eb59d25128c980895_b258fbd035e77c2729be5c41f0c32984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742034_1210 (size=4469) 2024-11-26T11:15:33,756 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/a7c6cf86c7d94c2bad34666f8e27fbb0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a7c6cf86c7d94c2bad34666f8e27fbb0 2024-11-26T11:15:33,765 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/B of b258fbd035e77c2729be5c41f0c32984 into a7c6cf86c7d94c2bad34666f8e27fbb0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:33,765 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:33,765 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/B, priority=13, startTime=1732619733334; duration=0sec 2024-11-26T11:15:33,765 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:33,765 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:B 2024-11-26T11:15:33,765 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:33,767 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:33,768 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:33,768 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): b258fbd035e77c2729be5c41f0c32984/C is initiating minor compaction (all files) 2024-11-26T11:15:33,768 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b258fbd035e77c2729be5c41f0c32984/C in TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:33,768 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/5cb7ba3dc9674d0ab54cb8c7c98b63e4 is 175, key is test_row_0/A:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:33,768 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/217cb3ee59bf490ba9b81fd011b87585, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4ed6f2f7b7614b59bba1e49a61ebb3f7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/c91b2e53fa754b07987ab56ba82f20df] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp, totalSize=36.8 K 2024-11-26T11:15:33,769 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 217cb3ee59bf490ba9b81fd011b87585, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732619728243 2024-11-26T11:15:33,770 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed6f2f7b7614b59bba1e49a61ebb3f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732619728321 2024-11-26T11:15:33,770 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c91b2e53fa754b07987ab56ba82f20df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732619729491 2024-11-26T11:15:33,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742035_1211 (size=32107) 2024-11-26T11:15:33,780 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b258fbd035e77c2729be5c41f0c32984#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:33,781 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/6c6cd104b0154356875a661559282d05 is 50, key is test_row_0/C:col10/1732619731643/Put/seqid=0 2024-11-26T11:15:33,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742036_1212 (size=13153) 2024-11-26T11:15:34,183 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/5cb7ba3dc9674d0ab54cb8c7c98b63e4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5cb7ba3dc9674d0ab54cb8c7c98b63e4 2024-11-26T11:15:34,191 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/A of b258fbd035e77c2729be5c41f0c32984 into 5cb7ba3dc9674d0ab54cb8c7c98b63e4(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:34,191 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:34,191 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/A, priority=13, startTime=1732619733333; duration=0sec 2024-11-26T11:15:34,191 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:34,192 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:A 2024-11-26T11:15:34,192 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/6c6cd104b0154356875a661559282d05 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/6c6cd104b0154356875a661559282d05 2024-11-26T11:15:34,197 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b258fbd035e77c2729be5c41f0c32984/C of b258fbd035e77c2729be5c41f0c32984 into 6c6cd104b0154356875a661559282d05(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:34,197 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b258fbd035e77c2729be5c41f0c32984: 2024-11-26T11:15:34,197 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., storeName=b258fbd035e77c2729be5c41f0c32984/C, priority=13, startTime=1732619733334; duration=0sec 2024-11-26T11:15:34,197 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:34,197 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b258fbd035e77c2729be5c41f0c32984:C 2024-11-26T11:15:34,263 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T11:15:40,154 INFO [Thread-568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-568. 2024-11-26T11:15:40,154 DEBUG [Thread-568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:15:40,154 DEBUG [Thread-568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:40,155 DEBUG [Thread-568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:40,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3147 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3248 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1364 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4092 rows 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1365 2024-11-26T11:15:40,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4093 rows 2024-11-26T11:15:40,155 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:15:40,155 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:15:40,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:40,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:40,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:15:40,156 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T11:15:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T11:15:40,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619740159"}]},"ts":"1732619740159"} 2024-11-26T11:15:40,161 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T11:15:40,161 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T11:15:40,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:15:40,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, UNASSIGN}] 2024-11-26T11:15:40,163 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, UNASSIGN 2024-11-26T11:15:40,164 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:40,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, UNASSIGN because future has completed 2024-11-26T11:15:40,165 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:15:40,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:15:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T11:15:40,317 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing b258fbd035e77c2729be5c41f0c32984, disabling compactions & flushes 2024-11-26T11:15:40,317 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. after waiting 0 ms 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:40,317 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing b258fbd035e77c2729be5c41f0c32984 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=A 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=B 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b258fbd035e77c2729be5c41f0c32984, store=C 2024-11-26T11:15:40,317 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:40,324 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126657448666216406a945b6524968907cc_b258fbd035e77c2729be5c41f0c32984 is 50, key is test_row_0/A:col10/1732619731673/Put/seqid=0 2024-11-26T11:15:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742037_1213 (size=9914) 2024-11-26T11:15:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T11:15:40,731 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:40,740 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126657448666216406a945b6524968907cc_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126657448666216406a945b6524968907cc_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:40,741 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8588bb074694447fbff74c2a4c7e9d38, store: [table=TestAcidGuarantees family=A region=b258fbd035e77c2729be5c41f0c32984] 2024-11-26T11:15:40,742 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8588bb074694447fbff74c2a4c7e9d38 is 175, key is test_row_0/A:col10/1732619731673/Put/seqid=0 2024-11-26T11:15:40,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742038_1214 (size=22561) 2024-11-26T11:15:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T11:15:41,146 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8588bb074694447fbff74c2a4c7e9d38 2024-11-26T11:15:41,153 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/109f715b1b3149138064804584282b80 is 50, key is test_row_0/B:col10/1732619731673/Put/seqid=0 2024-11-26T11:15:41,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742039_1215 (size=9857) 2024-11-26T11:15:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T11:15:41,558 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/109f715b1b3149138064804584282b80 2024-11-26T11:15:41,568 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8fbe260a8a1c4f248cf9137d3e76931b is 50, key is test_row_0/C:col10/1732619731673/Put/seqid=0 2024-11-26T11:15:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742040_1216 (size=9857) 2024-11-26T11:15:41,978 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8fbe260a8a1c4f248cf9137d3e76931b 2024-11-26T11:15:41,983 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/A/8588bb074694447fbff74c2a4c7e9d38 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8588bb074694447fbff74c2a4c7e9d38 2024-11-26T11:15:41,987 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8588bb074694447fbff74c2a4c7e9d38, entries=100, sequenceid=371, filesize=22.0 K 2024-11-26T11:15:41,988 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/B/109f715b1b3149138064804584282b80 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/109f715b1b3149138064804584282b80 2024-11-26T11:15:41,994 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/109f715b1b3149138064804584282b80, entries=100, sequenceid=371, filesize=9.6 K 2024-11-26T11:15:41,995 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/.tmp/C/8fbe260a8a1c4f248cf9137d3e76931b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8fbe260a8a1c4f248cf9137d3e76931b 2024-11-26T11:15:42,000 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8fbe260a8a1c4f248cf9137d3e76931b, entries=100, sequenceid=371, filesize=9.6 K 2024-11-26T11:15:42,001 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b258fbd035e77c2729be5c41f0c32984 in 1684ms, sequenceid=371, compaction requested=false 2024-11-26T11:15:42,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8fb487a9bc8b403fa13786627f0e0901, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/f85340627413484e9144d3a941fa04f7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/12804de0664945ba95ea2299bb658889, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0d26e5c88dce491a882fc9d45408455a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/720308c32bdd4c9bbf88c714e9601c98, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/e78aceccba654f10bf99653c84fd98d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/30c9a7c5412c4621b3ca42fcdcef5056, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e] to archive 2024-11-26T11:15:42,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:15:42,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9a83c28bb53e497cab950a4719ed1b75 2024-11-26T11:15:42,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/142169c966a945a0aaaf78f3f43307b8 2024-11-26T11:15:42,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5f220d641b4c4e2598251486e108bcf8 2024-11-26T11:15:42,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c9f6b2d9c5764c9cb7acc1ddbc4d5a42 2024-11-26T11:15:42,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8fb487a9bc8b403fa13786627f0e0901 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8fb487a9bc8b403fa13786627f0e0901 2024-11-26T11:15:42,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/bf1ce246a8d94998b5a0a5ff44ca2900 2024-11-26T11:15:42,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/6de2004e207e4073a8a8a36b4910393d 2024-11-26T11:15:42,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/f85340627413484e9144d3a941fa04f7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/f85340627413484e9144d3a941fa04f7 2024-11-26T11:15:42,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8126293f831b4be5a774ca0865ff4ca5 2024-11-26T11:15:42,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/546887f805294548bd206439b8f72f8b 2024-11-26T11:15:42,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/12804de0664945ba95ea2299bb658889 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/12804de0664945ba95ea2299bb658889 2024-11-26T11:15:42,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0e86bb9a57c44694b88c2f29ec302bb7 2024-11-26T11:15:42,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0d26e5c88dce491a882fc9d45408455a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/0d26e5c88dce491a882fc9d45408455a 2024-11-26T11:15:42,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/fe78607ceb3248a9911e289fc98e2dbc 2024-11-26T11:15:42,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/7b6a274c86934b03bd77a23729d9c5a8 2024-11-26T11:15:42,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/720308c32bdd4c9bbf88c714e9601c98 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/720308c32bdd4c9bbf88c714e9601c98 2024-11-26T11:15:42,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9aa9cb4338d140989f413da10358f06d 2024-11-26T11:15:42,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/a33f7129f0bc4acebb0c66fc1b7087c2 2024-11-26T11:15:42,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/00e2479d5e3847e2ab449d21d10c12ca 2024-11-26T11:15:42,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/e78aceccba654f10bf99653c84fd98d6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/e78aceccba654f10bf99653c84fd98d6 2024-11-26T11:15:42,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/c5ba2b6955da417a9c20dfe48f8a5261 2024-11-26T11:15:42,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/ef3da29e865c4431ae9f82e773e0fe44 2024-11-26T11:15:42,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/30c9a7c5412c4621b3ca42fcdcef5056 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/30c9a7c5412c4621b3ca42fcdcef5056 2024-11-26T11:15:42,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/d5b00eba17cf4ef3a4777ffe78d2b9ec 2024-11-26T11:15:42,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/9c018ad28115420faed913ffd34d711e 2024-11-26T11:15:42,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3d7c8126fa674112be85f07f3d3211e0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1000f4abf6f5450cafccf841a4c1d362, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/b32b06e8cd7d4023a75b2bbb0a7e3b80, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/2e2053a5fd574e5dbecc3a261451ca21, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3e43e08efc8e4eaf86cce8590665c5cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1e5017d792744ffa8c3db049bd2045b8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f11abfd03ece4ebfa1091c97be223a1d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/177d30f9cca542b48b32af711e066fbb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/7a832c24c08e40cca5da36c7edc331b3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/220f033b24e14438bd4f4c37c32fbf0e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/db9ba42ea4924b7586a73274f78d5df8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fbb293b9297847e58bad2a5a7ad0a0cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/4246c1f3daf447adb96ab7f30a145020, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/777de404deab4a87821202600edb9842, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/8502294202d4429aa86e445c78b9b601, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c6a38bf4d1fc4bd2a6fd72aca1048954, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/0b39e534742f44c189072ddbc6f87a96, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/307b4828c2bb474c96f5c2000d246207, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/56f6d9ecbe0e4038a2ae14a410d147e4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a31b41965e01408f828cdd82b5157308, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c8ee46ee692a40bf8e4b84e18b622365, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/6c56b2a4befc4ce6b42ac7798dfb6c68, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/17169272ae494124ab8e59ae2f9ffa89, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f65d8f3b6454490abd4231188aa3c458, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fe38244635ee4e8fbecf614d31c1db91] to archive 2024-11-26T11:15:42,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:15:42,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3d7c8126fa674112be85f07f3d3211e0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3d7c8126fa674112be85f07f3d3211e0 2024-11-26T11:15:42,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1000f4abf6f5450cafccf841a4c1d362 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1000f4abf6f5450cafccf841a4c1d362 2024-11-26T11:15:42,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/b32b06e8cd7d4023a75b2bbb0a7e3b80 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/b32b06e8cd7d4023a75b2bbb0a7e3b80 2024-11-26T11:15:42,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/2e2053a5fd574e5dbecc3a261451ca21 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/2e2053a5fd574e5dbecc3a261451ca21 2024-11-26T11:15:42,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3e43e08efc8e4eaf86cce8590665c5cd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/3e43e08efc8e4eaf86cce8590665c5cd 2024-11-26T11:15:42,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1e5017d792744ffa8c3db049bd2045b8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/1e5017d792744ffa8c3db049bd2045b8 2024-11-26T11:15:42,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f11abfd03ece4ebfa1091c97be223a1d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f11abfd03ece4ebfa1091c97be223a1d 2024-11-26T11:15:42,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/177d30f9cca542b48b32af711e066fbb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/177d30f9cca542b48b32af711e066fbb 2024-11-26T11:15:42,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/7a832c24c08e40cca5da36c7edc331b3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/7a832c24c08e40cca5da36c7edc331b3 2024-11-26T11:15:42,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/220f033b24e14438bd4f4c37c32fbf0e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/220f033b24e14438bd4f4c37c32fbf0e 2024-11-26T11:15:42,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/db9ba42ea4924b7586a73274f78d5df8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/db9ba42ea4924b7586a73274f78d5df8 2024-11-26T11:15:42,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fbb293b9297847e58bad2a5a7ad0a0cd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fbb293b9297847e58bad2a5a7ad0a0cd 2024-11-26T11:15:42,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/4246c1f3daf447adb96ab7f30a145020 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/4246c1f3daf447adb96ab7f30a145020 2024-11-26T11:15:42,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/777de404deab4a87821202600edb9842 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/777de404deab4a87821202600edb9842 2024-11-26T11:15:42,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/8502294202d4429aa86e445c78b9b601 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/8502294202d4429aa86e445c78b9b601 2024-11-26T11:15:42,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c6a38bf4d1fc4bd2a6fd72aca1048954 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c6a38bf4d1fc4bd2a6fd72aca1048954 2024-11-26T11:15:42,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/0b39e534742f44c189072ddbc6f87a96 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/0b39e534742f44c189072ddbc6f87a96 2024-11-26T11:15:42,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/307b4828c2bb474c96f5c2000d246207 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/307b4828c2bb474c96f5c2000d246207 2024-11-26T11:15:42,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/56f6d9ecbe0e4038a2ae14a410d147e4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/56f6d9ecbe0e4038a2ae14a410d147e4 2024-11-26T11:15:42,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a31b41965e01408f828cdd82b5157308 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a31b41965e01408f828cdd82b5157308 2024-11-26T11:15:42,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c8ee46ee692a40bf8e4b84e18b622365 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/c8ee46ee692a40bf8e4b84e18b622365 2024-11-26T11:15:42,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/6c56b2a4befc4ce6b42ac7798dfb6c68 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/6c56b2a4befc4ce6b42ac7798dfb6c68 2024-11-26T11:15:42,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/17169272ae494124ab8e59ae2f9ffa89 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/17169272ae494124ab8e59ae2f9ffa89 2024-11-26T11:15:42,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f65d8f3b6454490abd4231188aa3c458 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/f65d8f3b6454490abd4231188aa3c458 2024-11-26T11:15:42,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fe38244635ee4e8fbecf614d31c1db91 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/fe38244635ee4e8fbecf614d31c1db91 2024-11-26T11:15:42,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ad8ed8727f2548a8be6ad2437b3983e8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/948f3a6bd3da479e9c8e5081c335ab8f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4c273c09bbf14609b57a1fa4fe9a6e49, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ae04ee4be321466299b6ad7fdf59382d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/41e7f9ea68414a668a66c32dc0b1f3ea, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/17b9835a42df4d5890fa0ba1225f237c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ce2e196a84c54848aa27c8b4f70714f2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46a52cb47ede4b88bf5e53326d4ef6e2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1a73cee5cbe5478ba58dca90d1110b23, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/728e286db084406580e3076e0e0247ee, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8e28ee31bf7642bfbc063b0d0c6510e2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/23e3e3bf5a094c1a979b36e4176157e1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46f4c760855c441d982633af67474054, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8158f8588f5642678563d317c8269000, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/e52d2795e3f24d03a1f64cefc85d2ece, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bf75544f16bd47f1af807c98919169bc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1119fe710f6345178decf1cfe190f9f0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bd631c7e9ab942569b4ce70fe61153fb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3ab835bf552649e680804ca68e8ee75a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/20b20c7ff3984a9f98b475465a665385, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/f2e0f08a0b304ffdb15a0856757ab8ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/217cb3ee59bf490ba9b81fd011b87585, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3c8b628efa0143f2835685ee00367cea, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4ed6f2f7b7614b59bba1e49a61ebb3f7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/c91b2e53fa754b07987ab56ba82f20df] to archive 2024-11-26T11:15:42,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:15:42,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ad8ed8727f2548a8be6ad2437b3983e8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ad8ed8727f2548a8be6ad2437b3983e8 2024-11-26T11:15:42,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/948f3a6bd3da479e9c8e5081c335ab8f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/948f3a6bd3da479e9c8e5081c335ab8f 2024-11-26T11:15:42,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4c273c09bbf14609b57a1fa4fe9a6e49 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4c273c09bbf14609b57a1fa4fe9a6e49 2024-11-26T11:15:42,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ae04ee4be321466299b6ad7fdf59382d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ae04ee4be321466299b6ad7fdf59382d 2024-11-26T11:15:42,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/41e7f9ea68414a668a66c32dc0b1f3ea to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/41e7f9ea68414a668a66c32dc0b1f3ea 2024-11-26T11:15:42,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/17b9835a42df4d5890fa0ba1225f237c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/17b9835a42df4d5890fa0ba1225f237c 2024-11-26T11:15:42,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ce2e196a84c54848aa27c8b4f70714f2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/ce2e196a84c54848aa27c8b4f70714f2 2024-11-26T11:15:42,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46a52cb47ede4b88bf5e53326d4ef6e2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46a52cb47ede4b88bf5e53326d4ef6e2 2024-11-26T11:15:42,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1a73cee5cbe5478ba58dca90d1110b23 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1a73cee5cbe5478ba58dca90d1110b23 2024-11-26T11:15:42,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/728e286db084406580e3076e0e0247ee to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/728e286db084406580e3076e0e0247ee 2024-11-26T11:15:42,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8e28ee31bf7642bfbc063b0d0c6510e2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8e28ee31bf7642bfbc063b0d0c6510e2 2024-11-26T11:15:42,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/23e3e3bf5a094c1a979b36e4176157e1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/23e3e3bf5a094c1a979b36e4176157e1 2024-11-26T11:15:42,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46f4c760855c441d982633af67474054 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/46f4c760855c441d982633af67474054 2024-11-26T11:15:42,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8158f8588f5642678563d317c8269000 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8158f8588f5642678563d317c8269000 2024-11-26T11:15:42,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/e52d2795e3f24d03a1f64cefc85d2ece to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/e52d2795e3f24d03a1f64cefc85d2ece 2024-11-26T11:15:42,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bf75544f16bd47f1af807c98919169bc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bf75544f16bd47f1af807c98919169bc 2024-11-26T11:15:42,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1119fe710f6345178decf1cfe190f9f0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/1119fe710f6345178decf1cfe190f9f0 2024-11-26T11:15:42,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bd631c7e9ab942569b4ce70fe61153fb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/bd631c7e9ab942569b4ce70fe61153fb 2024-11-26T11:15:42,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3ab835bf552649e680804ca68e8ee75a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3ab835bf552649e680804ca68e8ee75a 2024-11-26T11:15:42,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/20b20c7ff3984a9f98b475465a665385 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/20b20c7ff3984a9f98b475465a665385 2024-11-26T11:15:42,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/f2e0f08a0b304ffdb15a0856757ab8ff to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/f2e0f08a0b304ffdb15a0856757ab8ff 2024-11-26T11:15:42,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/217cb3ee59bf490ba9b81fd011b87585 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/217cb3ee59bf490ba9b81fd011b87585 2024-11-26T11:15:42,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3c8b628efa0143f2835685ee00367cea to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/3c8b628efa0143f2835685ee00367cea 2024-11-26T11:15:42,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4ed6f2f7b7614b59bba1e49a61ebb3f7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/4ed6f2f7b7614b59bba1e49a61ebb3f7 2024-11-26T11:15:42,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/c91b2e53fa754b07987ab56ba82f20df to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/c91b2e53fa754b07987ab56ba82f20df 2024-11-26T11:15:42,112 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/recovered.edits/374.seqid, newMaxSeqId=374, maxSeqId=4 2024-11-26T11:15:42,113 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. 2024-11-26T11:15:42,114 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for b258fbd035e77c2729be5c41f0c32984: Waiting for close lock at 1732619740317Running coprocessor pre-close hooks at 1732619740317Disabling compacts and flushes for region at 1732619740317Disabling writes for close at 1732619740317Obtaining lock to block concurrent updates at 1732619740317Preparing flush snapshotting stores in b258fbd035e77c2729be5c41f0c32984 at 1732619740317Finished memstore snapshotting TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732619740317Flushing stores of TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984. at 1732619740318 (+1 ms)Flushing b258fbd035e77c2729be5c41f0c32984/A: creating writer at 1732619740318Flushing mob file b258fbd035e77c2729be5c41f0c32984/A: appending metadata at 1732619740323 (+5 ms)Flushing mob file b258fbd035e77c2729be5c41f0c32984/A: closing flushed file at 1732619740323Flushing b258fbd035e77c2729be5c41f0c32984/A: appending metadata at 1732619740741 (+418 ms)Flushing b258fbd035e77c2729be5c41f0c32984/A: closing flushed file at 1732619740741Flushing b258fbd035e77c2729be5c41f0c32984/B: creating writer at 1732619741150 (+409 ms)Flushing b258fbd035e77c2729be5c41f0c32984/B: appending metadata at 1732619741153 (+3 ms)Flushing b258fbd035e77c2729be5c41f0c32984/B: closing flushed file at 1732619741153Flushing b258fbd035e77c2729be5c41f0c32984/C: creating writer at 1732619741564 (+411 ms)Flushing b258fbd035e77c2729be5c41f0c32984/C: appending metadata at 1732619741568 (+4 ms)Flushing b258fbd035e77c2729be5c41f0c32984/C: closing flushed file at 1732619741568Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16e5f1e0: reopening flushed file at 1732619741982 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@672fc8f2: reopening flushed file at 1732619741987 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b7bbcb9: reopening flushed file at 1732619741994 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b258fbd035e77c2729be5c41f0c32984 in 1684ms, sequenceid=371, compaction requested=false at 1732619742001 (+7 ms)Writing region close event to WAL at 1732619742108 (+107 ms)Running coprocessor post-close hooks at 1732619742113 (+5 ms)Closed at 1732619742113 2024-11-26T11:15:42,116 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,117 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=b258fbd035e77c2729be5c41f0c32984, regionState=CLOSED 2024-11-26T11:15:42,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:15:42,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T11:15:42,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure b258fbd035e77c2729be5c41f0c32984, server=7f3a3edaecc3,37499,1732619676667 in 1.9550 sec 2024-11-26T11:15:42,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-26T11:15:42,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b258fbd035e77c2729be5c41f0c32984, UNASSIGN in 1.9600 sec 2024-11-26T11:15:42,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-26T11:15:42,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9650 sec 2024-11-26T11:15:42,130 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619742130"}]},"ts":"1732619742130"} 2024-11-26T11:15:42,132 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T11:15:42,132 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T11:15:42,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9770 sec 2024-11-26T11:15:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T11:15:42,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:42,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T11:15:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,306 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T11:15:42,307 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,310 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,314 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/recovered.edits] 2024-11-26T11:15:42,317 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5cb7ba3dc9674d0ab54cb8c7c98b63e4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/5cb7ba3dc9674d0ab54cb8c7c98b63e4 2024-11-26T11:15:42,318 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8588bb074694447fbff74c2a4c7e9d38 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/A/8588bb074694447fbff74c2a4c7e9d38 2024-11-26T11:15:42,321 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/109f715b1b3149138064804584282b80 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/109f715b1b3149138064804584282b80 2024-11-26T11:15:42,323 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a7c6cf86c7d94c2bad34666f8e27fbb0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/B/a7c6cf86c7d94c2bad34666f8e27fbb0 2024-11-26T11:15:42,326 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/6c6cd104b0154356875a661559282d05 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/6c6cd104b0154356875a661559282d05 2024-11-26T11:15:42,328 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8fbe260a8a1c4f248cf9137d3e76931b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/C/8fbe260a8a1c4f248cf9137d3e76931b 2024-11-26T11:15:42,333 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/recovered.edits/374.seqid to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984/recovered.edits/374.seqid 2024-11-26T11:15:42,334 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,334 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T11:15:42,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T11:15:42,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T11:15:42,341 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112613cd655def224a768bbb6bd15f80b2f0_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112613cd655def224a768bbb6bd15f80b2f0_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126189a5cfe2ef04968a7e59f09df4f04c7_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126189a5cfe2ef04968a7e59f09df4f04c7_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,343 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112621a15005d8054c7daeb4d33d02a64972_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112621a15005d8054c7daeb4d33d02a64972_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,345 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112627b40270bd4040d89f3100eae325f357_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112627b40270bd4040d89f3100eae325f357_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,346 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126283ac3eae6c84fe8aaeebd00d90974eb_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126283ac3eae6c84fe8aaeebd00d90974eb_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,348 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264084356fe468495f89144a572b531b9f_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264084356fe468495f89144a572b531b9f_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126454c5ef34dd6425c969250f1d4eb83b1_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126454c5ef34dd6425c969250f1d4eb83b1_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264c5292e8619d448684629f6f561ba52c_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264c5292e8619d448684629f6f561ba52c_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126580d2deaa2e149909998b2941c591d39_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126580d2deaa2e149909998b2941c591d39_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265894c3d815a74907937ea4fb078a2599_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265894c3d815a74907937ea4fb078a2599_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,354 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126657448666216406a945b6524968907cc_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126657448666216406a945b6524968907cc_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267a01bc9668cd49bc8fdb86cb93e69a03_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267a01bc9668cd49bc8fdb86cb93e69a03_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268ea5fa435ea1439bbac5f20e68c80396_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268ea5fa435ea1439bbac5f20e68c80396_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a2b2b39dc3034d569a04ae1c185fe998_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a2b2b39dc3034d569a04ae1c185fe998_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,360 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b6f86f093b6b44bb8a3760c268a2a66b_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b6f86f093b6b44bb8a3760c268a2a66b_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,363 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bb99f8405d624ef7b3d460b13f08d46b_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bb99f8405d624ef7b3d460b13f08d46b_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,364 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c4bc3fc717fd4d1db655169fb256283d_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c4bc3fc717fd4d1db655169fb256283d_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,365 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e4dfd78c3f5841b6967ba17e510b134d_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e4dfd78c3f5841b6967ba17e510b134d_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,367 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fa4a7fd6d70c4db29f65a91ba8254944_b258fbd035e77c2729be5c41f0c32984 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fa4a7fd6d70c4db29f65a91ba8254944_b258fbd035e77c2729be5c41f0c32984 2024-11-26T11:15:42,367 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T11:15:42,369 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,372 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T11:15:42,375 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T11:15:42,377 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,377 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T11:15:42,377 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732619742377"}]},"ts":"9223372036854775807"} 2024-11-26T11:15:42,380 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T11:15:42,380 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b258fbd035e77c2729be5c41f0c32984, NAME => 'TestAcidGuarantees,,1732619708352.b258fbd035e77c2729be5c41f0c32984.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T11:15:42,380 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T11:15:42,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732619742380"}]},"ts":"9223372036854775807"} 2024-11-26T11:15:42,383 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T11:15:42,384 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-26T11:15:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T11:15:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T11:15:42,414 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:42,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 371) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4089 (was 2867) - AvailableMemoryMB LEAK? - 2024-11-26T11:15:42,431 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=4088 2024-11-26T11:15:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:15:42,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:15:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T11:15:42,436 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T11:15:42,436 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:42,436 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-26T11:15:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-26T11:15:42,437 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T11:15:42,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742041_1217 (size=1061) 2024-11-26T11:15:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-26T11:15:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-26T11:15:42,845 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:15:42,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742042_1218 (size=53) 2024-11-26T11:15:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-26T11:15:43,252 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:15:43,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing de67c03f4cecf266dfeecbd54a45d9f0, disabling compactions & flushes 2024-11-26T11:15:43,253 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. after waiting 0 ms 2024-11-26T11:15:43,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,253 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,253 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for de67c03f4cecf266dfeecbd54a45d9f0: Waiting for close lock at 1732619743253Disabling compacts and flushes for region at 1732619743253Disabling writes for close at 1732619743253Writing region close event to WAL at 1732619743253Closed at 1732619743253 2024-11-26T11:15:43,254 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T11:15:43,254 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732619743254"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732619743254"}]},"ts":"1732619743254"} 2024-11-26T11:15:43,256 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T11:15:43,257 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T11:15:43,257 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619743257"}]},"ts":"1732619743257"} 2024-11-26T11:15:43,258 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T11:15:43,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, ASSIGN}] 2024-11-26T11:15:43,259 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, ASSIGN 2024-11-26T11:15:43,260 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, ASSIGN; state=OFFLINE, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=false 2024-11-26T11:15:43,411 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=de67c03f4cecf266dfeecbd54a45d9f0, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:43,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, ASSIGN because future has completed 2024-11-26T11:15:43,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:15:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-26T11:15:43,574 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,575 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:15:43,576 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,576 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:15:43,577 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,577 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,580 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,581 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:43,582 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de67c03f4cecf266dfeecbd54a45d9f0 columnFamilyName A 2024-11-26T11:15:43,582 DEBUG [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:43,582 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.HStore(327): Store=de67c03f4cecf266dfeecbd54a45d9f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:43,582 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,583 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:43,584 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de67c03f4cecf266dfeecbd54a45d9f0 columnFamilyName B 2024-11-26T11:15:43,584 DEBUG [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:43,584 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.HStore(327): Store=de67c03f4cecf266dfeecbd54a45d9f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:43,584 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,585 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:15:43,585 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de67c03f4cecf266dfeecbd54a45d9f0 columnFamilyName C 2024-11-26T11:15:43,585 DEBUG [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:15:43,586 INFO [StoreOpener-de67c03f4cecf266dfeecbd54a45d9f0-1 {}] regionserver.HStore(327): Store=de67c03f4cecf266dfeecbd54a45d9f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:15:43,586 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,586 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,587 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,587 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,588 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,588 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,588 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:15:43,589 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,591 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:15:43,591 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened de67c03f4cecf266dfeecbd54a45d9f0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74422410, jitterRate=0.10898032784461975}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:15:43,591 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:43,592 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for de67c03f4cecf266dfeecbd54a45d9f0: Running coprocessor pre-open hook at 1732619743577Writing region info on filesystem at 1732619743577Initializing all the Stores at 1732619743579 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619743579Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619743580 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619743580Cleaning up temporary data from old regions at 1732619743588 (+8 ms)Running coprocessor post-open hooks at 1732619743591 (+3 ms)Region opened successfully at 1732619743592 (+1 ms) 2024-11-26T11:15:43,592 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., pid=55, masterSystemTime=1732619743567 2024-11-26T11:15:43,594 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,594 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:43,595 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=de67c03f4cecf266dfeecbd54a45d9f0, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:43,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:15:43,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T11:15:43,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 in 183 msec 2024-11-26T11:15:43,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-26T11:15:43,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, ASSIGN in 340 msec 2024-11-26T11:15:43,601 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T11:15:43,601 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619743601"}]},"ts":"1732619743601"} 2024-11-26T11:15:43,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T11:15:43,603 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T11:15:43,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-26T11:15:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-26T11:15:44,583 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:44,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,586 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519871e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,587 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4a80f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,590 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:15:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:15:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@19848f33 2024-11-26T11:15:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:15:44,592 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:15:44,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,594 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9decfe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b4e4cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,598 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161ea75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,599 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cef6b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,601 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16960267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306c02cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,610 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b23232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,612 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335364c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53daaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,616 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158a30e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,618 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@137e3ffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,620 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a40ca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544bf67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@410ec251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,626 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2089b81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8b88f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,631 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d023a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc44b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:15:44,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:15:44,635 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:15:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:15:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:15:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2961ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:15:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:15:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:15:44,637 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:15:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f7c581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:15:44,638 DEBUG [Thread-1034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:44,639 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,639 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,640 DEBUG [Thread-1030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-26T11:15:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,641 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:44,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:44,642 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:44,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:44,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:44,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:44,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:44,660 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,661 DEBUG [Thread-1047 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,662 DEBUG [Thread-1047 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/6c4a18364e464f029a46dffbe9f620c6 is 50, key is test_row_0/A:col10/1732619744650/Put/seqid=0 2024-11-26T11:15:44,681 DEBUG [Thread-1032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,682 DEBUG [Thread-1038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,684 DEBUG [Thread-1038 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,684 DEBUG [Thread-1038 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,696 DEBUG [Thread-1049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,698 DEBUG [Thread-1049 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732619744661, max timestamp=1732619744694Num uniques 859; 2024-11-26T11:15:44,698 DEBUG [Thread-1049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-26T11:15:44,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732619744661, max timestamp=1732619744694Num uniques 859; 2024-11-26T11:15:44,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-26T11:15:44,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619744679, max timestamp=1732619744694Num uniques 859; 2024-11-26T11:15:44,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:44,736 DEBUG [Thread-1036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619804733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619804735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,740 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:15:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:15:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:15:44,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:15:44,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619804737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619804738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619804742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:15:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:44,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742043_1219 (size=12001) 2024-11-26T11:15:44,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/6c4a18364e464f029a46dffbe9f620c6 2024-11-26T11:15:44,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:44,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:44,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/dd283a0d38814e538a22c42a4cb0d4f0 is 50, key is test_row_0/B:col10/1732619744650/Put/seqid=0 2024-11-26T11:15:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619804843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619804843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742044_1220 (size=12001) 2024-11-26T11:15:44,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/dd283a0d38814e538a22c42a4cb0d4f0 2024-11-26T11:15:44,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619804853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619804853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619804853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:44,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/e28f51417c9545baaa5a3515ba0691c4 is 50, key is test_row_0/C:col10/1732619744650/Put/seqid=0 2024-11-26T11:15:44,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742045_1221 (size=12001) 2024-11-26T11:15:44,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/e28f51417c9545baaa5a3515ba0691c4 2024-11-26T11:15:44,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/6c4a18364e464f029a46dffbe9f620c6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/6c4a18364e464f029a46dffbe9f620c6 2024-11-26T11:15:44,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/6c4a18364e464f029a46dffbe9f620c6, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T11:15:44,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/dd283a0d38814e538a22c42a4cb0d4f0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/dd283a0d38814e538a22c42a4cb0d4f0 2024-11-26T11:15:44,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/dd283a0d38814e538a22c42a4cb0d4f0, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T11:15:44,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:44,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:44,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/e28f51417c9545baaa5a3515ba0691c4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/e28f51417c9545baaa5a3515ba0691c4 2024-11-26T11:15:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:44,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/e28f51417c9545baaa5a3515ba0691c4, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T11:15:44,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=145.36 KB/148850 for de67c03f4cecf266dfeecbd54a45d9f0 in 307ms, sequenceid=11, compaction requested=false 2024-11-26T11:15:44,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:45,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=152.07 KB heapSize=416.23 KB 2024-11-26T11:15:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:45,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:45,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:45,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:45,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:45,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:45,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/162c36bb5289458aa2a9945acd58466b is 50, key is test_row_0/A:col10/1732619745055/Put/seqid=0 2024-11-26T11:15:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619805072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619805073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619805073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619805074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619805078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742046_1222 (size=12001) 2024-11-26T11:15:45,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/162c36bb5289458aa2a9945acd58466b 2024-11-26T11:15:45,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/2c90743107964b068576c6b13e7a2add is 50, key is test_row_0/B:col10/1732619745055/Put/seqid=0 2024-11-26T11:15:45,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742047_1223 (size=12001) 2024-11-26T11:15:45,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/2c90743107964b068576c6b13e7a2add 2024-11-26T11:15:45,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/319573b947f445aea15b0550ae111dc7 is 50, key is test_row_0/C:col10/1732619745055/Put/seqid=0 2024-11-26T11:15:45,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742048_1224 (size=12001) 2024-11-26T11:15:45,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619805193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619805203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:45,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:45,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619805382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619805382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619805393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619805403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:45,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:45,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619805413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:45,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:45,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:45,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/319573b947f445aea15b0550ae111dc7 2024-11-26T11:15:45,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/162c36bb5289458aa2a9945acd58466b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/162c36bb5289458aa2a9945acd58466b 2024-11-26T11:15:45,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/162c36bb5289458aa2a9945acd58466b, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T11:15:45,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/2c90743107964b068576c6b13e7a2add as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/2c90743107964b068576c6b13e7a2add 2024-11-26T11:15:45,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/2c90743107964b068576c6b13e7a2add, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T11:15:45,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/319573b947f445aea15b0550ae111dc7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/319573b947f445aea15b0550ae111dc7 2024-11-26T11:15:45,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/319573b947f445aea15b0550ae111dc7, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T11:15:45,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~416.19 KB/426176, currentSize=33.54 KB/34350 for de67c03f4cecf266dfeecbd54a45d9f0 in 556ms, sequenceid=46, compaction requested=false 2024-11-26T11:15:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:45,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T11:15:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:45,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T11:15:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:45,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:45,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/9c4c8f0291c343d2991fcb606de58fbd is 50, key is test_row_0/A:col10/1732619745075/Put/seqid=0 2024-11-26T11:15:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742049_1225 (size=12001) 2024-11-26T11:15:45,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619745780, max timestamp=1732619745815Num uniques 859; 2024-11-26T11:15:45,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619745780, max timestamp=1732619745815Num uniques 859; 2024-11-26T11:15:45,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:45,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:45,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619745780, max timestamp=1732619745815Num uniques 859; 2024-11-26T11:15:45,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:45,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619745860, max timestamp=1732619745902Num uniques 1018; 2024-11-26T11:15:45,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:15:45,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619745860, max timestamp=1732619745902Num uniques 1018; 2024-11-26T11:15:45,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:15:45,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619745860, max timestamp=1732619745902Num uniques 1018; 2024-11-26T11:15:45,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:15:45,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619745936, max timestamp=1732619745951Num uniques 1077; 2024-11-26T11:15:45,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619745936, max timestamp=1732619745951Num uniques 1077; 2024-11-26T11:15:45,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T11:15:45,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T11:15:45,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619745936, max timestamp=1732619745951Num uniques 1077; 2024-11-26T11:15:45,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T11:15:45,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619805941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619805993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619805995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619805998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619806000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619806062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619806113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619806113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619806114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619806114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,186 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/9c4c8f0291c343d2991fcb606de58fbd 2024-11-26T11:15:46,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/726f3857e6624ef39403f9fa3f0676c5 is 50, key is test_row_0/B:col10/1732619745075/Put/seqid=0 2024-11-26T11:15:46,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742050_1226 (size=12001) 2024-11-26T11:15:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619806273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619806322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619806322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619806322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619806323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619806583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/726f3857e6624ef39403f9fa3f0676c5 2024-11-26T11:15:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/146c9d144d7c428d8d2925209b0c9e6b is 50, key is test_row_0/C:col10/1732619745075/Put/seqid=0 2024-11-26T11:15:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742051_1227 (size=12001) 2024-11-26T11:15:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619806633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619806633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619806633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619806633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:47,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/146c9d144d7c428d8d2925209b0c9e6b 2024-11-26T11:15:47,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/9c4c8f0291c343d2991fcb606de58fbd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/9c4c8f0291c343d2991fcb606de58fbd 2024-11-26T11:15:47,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/9c4c8f0291c343d2991fcb606de58fbd, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T11:15:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/726f3857e6624ef39403f9fa3f0676c5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/726f3857e6624ef39403f9fa3f0676c5 2024-11-26T11:15:47,053 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/726f3857e6624ef39403f9fa3f0676c5, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T11:15:47,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/146c9d144d7c428d8d2925209b0c9e6b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/146c9d144d7c428d8d2925209b0c9e6b 2024-11-26T11:15:47,059 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/146c9d144d7c428d8d2925209b0c9e6b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-26T11:15:47,060 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for de67c03f4cecf266dfeecbd54a45d9f0 in 1340ms, sequenceid=55, compaction requested=true 2024-11-26T11:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-26T11:15:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-26T11:15:47,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T11:15:47,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4190 sec 2024-11-26T11:15:47,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.4250 sec 2024-11-26T11:15:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:47,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732619745985, max timestamp=1732619747094Num uniques 1086; 2024-11-26T11:15:47,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T11:15:47,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732619745985, max timestamp=1732619747094Num uniques 1086; 2024-11-26T11:15:47,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T11:15:47,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732619745985, max timestamp=1732619747094Num uniques 1086; 2024-11-26T11:15:47,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=67.09 KB heapSize=182.19 KB 2024-11-26T11:15:47,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T11:15:47,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:47,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:47,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:47,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/eef6152e325d4855abc111984eca994d is 50, key is test_row_0/A:col10/1732619745993/Put/seqid=0 2024-11-26T11:15:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742052_1228 (size=12001) 2024-11-26T11:15:47,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619747146, max timestamp=1732619747156Num uniques 859; 2024-11-26T11:15:47,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619747146, max timestamp=1732619747156Num uniques 859; 2024-11-26T11:15:47,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619747146, max timestamp=1732619747156Num uniques 859; 2024-11-26T11:15:47,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:47,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:47,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:15:47,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732619747166, max timestamp=1732619747183Num uniques 1118; 2024-11-26T11:15:47,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732619747166, max timestamp=1732619747183Num uniques 1118; 2024-11-26T11:15:47,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-26T11:15:47,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-26T11:15:47,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732619747166, max timestamp=1732619747183Num uniques 1118; 2024-11-26T11:15:47,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-26T11:15:47,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747193, max timestamp=1732619747212Num uniques 1177; 2024-11-26T11:15:47,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-26T11:15:47,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747193, max timestamp=1732619747212Num uniques 1177; 2024-11-26T11:15:47,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-26T11:15:47,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747193, max timestamp=1732619747212Num uniques 1177; 2024-11-26T11:15:47,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-26T11:15:47,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619747224, max timestamp=1732619747242Num uniques 1086; 2024-11-26T11:15:47,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:47,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619747224, max timestamp=1732619747242Num uniques 1086; 2024-11-26T11:15:47,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:47,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619747224, max timestamp=1732619747242Num uniques 1086; 2024-11-26T11:15:47,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:47,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732619747264, max timestamp=1732619747270Num uniques 1095; 2024-11-26T11:15:47,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-26T11:15:47,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619747256, max timestamp=1732619747270Num uniques 1095; 2024-11-26T11:15:47,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T11:15:47,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732619747264, max timestamp=1732619747270Num uniques 1095; 2024-11-26T11:15:47,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-26T11:15:47,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732619747287, max timestamp=1732619747299Num uniques 1104; 2024-11-26T11:15:47,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732619747287, max timestamp=1732619747299Num uniques 1054; 2024-11-26T11:15:47,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732619747287, max timestamp=1732619747299Num uniques 1054; 2024-11-26T11:15:47,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T11:15:47,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-26T11:15:47,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-26T11:15:47,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732619747314, max timestamp=1732619747326Num uniques 1063; 2024-11-26T11:15:47,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-11-26T11:15:47,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732619747314, max timestamp=1732619747326Num uniques 1063; 2024-11-26T11:15:47,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-11-26T11:15:47,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732619747314, max timestamp=1732619747326Num uniques 1063; 2024-11-26T11:15:47,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-11-26T11:15:47,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732619747376, max timestamp=1732619747386Num uniques 1122; 2024-11-26T11:15:47,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-26T11:15:47,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732619747376, max timestamp=1732619747386Num uniques 1122; 2024-11-26T11:15:47,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-26T11:15:47,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732619747376, max timestamp=1732619747386Num uniques 1122; 2024-11-26T11:15:47,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-11-26T11:15:47,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1732619747408, max timestamp=1732619747418Num uniques 1031; 2024-11-26T11:15:47,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=15156, suffix heap size=135192,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=331 2024-11-26T11:15:47,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1732619747408, max timestamp=1732619747418Num uniques 1031; 2024-11-26T11:15:47,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=15156, suffix heap size=135192,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=331 2024-11-26T11:15:47,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732619747408, max timestamp=1732619747418Num uniques 1031; 2024-11-26T11:15:47,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-26T11:15:47,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732619747440, max timestamp=1732619747448Num uniques 1190; 2024-11-26T11:15:47,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732619747440, max timestamp=1732619747448Num uniques 1190; 2024-11-26T11:15:47,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=8700, suffix heap size=151728,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=190 2024-11-26T11:15:47,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=8700, suffix heap size=151728,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=190 2024-11-26T11:15:47,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732619747440, max timestamp=1732619747448Num uniques 1090; 2024-11-26T11:15:47,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-26T11:15:47,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/eef6152e325d4855abc111984eca994d 2024-11-26T11:15:47,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732619747509, max timestamp=1732619747515Num uniques 1049; 2024-11-26T11:15:47,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732619747509, max timestamp=1732619747515Num uniques 1049; 2024-11-26T11:15:47,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-26T11:15:47,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-26T11:15:47,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732619747509, max timestamp=1732619747517Num uniques 1049; 2024-11-26T11:15:47,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-26T11:15:47,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/4d8754ec38094dbfa1dcdcd3a69b9f68 is 50, key is test_row_0/B:col10/1732619745993/Put/seqid=0 2024-11-26T11:15:47,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742053_1229 (size=12001) 2024-11-26T11:15:47,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/4d8754ec38094dbfa1dcdcd3a69b9f68 2024-11-26T11:15:47,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/3dc7300a00b848f9ba1c1a56a95410b5 is 50, key is test_row_0/C:col10/1732619745993/Put/seqid=0 2024-11-26T11:15:47,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742054_1230 (size=12001) 2024-11-26T11:15:47,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/3dc7300a00b848f9ba1c1a56a95410b5 2024-11-26T11:15:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/eef6152e325d4855abc111984eca994d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eef6152e325d4855abc111984eca994d 2024-11-26T11:15:47,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eef6152e325d4855abc111984eca994d, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T11:15:47,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/4d8754ec38094dbfa1dcdcd3a69b9f68 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/4d8754ec38094dbfa1dcdcd3a69b9f68 2024-11-26T11:15:47,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/4d8754ec38094dbfa1dcdcd3a69b9f68, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T11:15:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/3dc7300a00b848f9ba1c1a56a95410b5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/3dc7300a00b848f9ba1c1a56a95410b5 2024-11-26T11:15:47,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/3dc7300a00b848f9ba1c1a56a95410b5, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T11:15:47,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~63.66 KB/65184, currentSize=120.76 KB/123660 for de67c03f4cecf266dfeecbd54a45d9f0 in 496ms, sequenceid=127, compaction requested=true 2024-11-26T11:15:47,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:47,595 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:47,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:47,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:47,595 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:47,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:47,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:47,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:47,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:47,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=140.89 KB heapSize=401.16 KB 2024-11-26T11:15:47,597 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:47,597 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:15:47,597 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:47,597 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/6c4a18364e464f029a46dffbe9f620c6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/162c36bb5289458aa2a9945acd58466b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/9c4c8f0291c343d2991fcb606de58fbd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eef6152e325d4855abc111984eca994d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=46.9 K 2024-11-26T11:15:47,598 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:47,598 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:15:47,598 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c4a18364e464f029a46dffbe9f620c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732619744647 2024-11-26T11:15:47,598 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:47,598 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/dd283a0d38814e538a22c42a4cb0d4f0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/2c90743107964b068576c6b13e7a2add, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/726f3857e6624ef39403f9fa3f0676c5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/4d8754ec38094dbfa1dcdcd3a69b9f68] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=46.9 K 2024-11-26T11:15:47,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 162c36bb5289458aa2a9945acd58466b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732619744726 2024-11-26T11:15:47,599 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd283a0d38814e538a22c42a4cb0d4f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732619744647 2024-11-26T11:15:47,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4c8f0291c343d2991fcb606de58fbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732619745058 2024-11-26T11:15:47,599 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c90743107964b068576c6b13e7a2add, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732619744726 2024-11-26T11:15:47,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting eef6152e325d4855abc111984eca994d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619745985 2024-11-26T11:15:47,599 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 726f3857e6624ef39403f9fa3f0676c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732619745058 2024-11-26T11:15:47,600 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8754ec38094dbfa1dcdcd3a69b9f68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619745985 2024-11-26T11:15:47,614 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#187 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:47,614 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/739fa900a7054c0681d2c0f229bed1f3 is 50, key is test_row_0/A:col10/1732619745993/Put/seqid=0 2024-11-26T11:15:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732619747547, max timestamp=1732619747595Num uniques 1058; 2024-11-26T11:15:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732619747547, max timestamp=1732619747595Num uniques 1058; 2024-11-26T11:15:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732619747547, max timestamp=1732619747595Num uniques 1058; 2024-11-26T11:15:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-26T11:15:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-26T11:15:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-26T11:15:47,617 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:47,618 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/0db090d4ef99420787bd6768394a364d is 50, key is test_row_0/B:col10/1732619745993/Put/seqid=0 2024-11-26T11:15:47,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:47,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:47,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:47,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742055_1231 (size=12139) 2024-11-26T11:15:47,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3b8de98287d14df8804992349b4f278c is 50, key is test_row_0/A:col10/1732619747549/Put/seqid=0 2024-11-26T11:15:47,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742056_1232 (size=12139) 2024-11-26T11:15:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742057_1233 (size=12301) 2024-11-26T11:15:47,639 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:15:47,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3b8de98287d14df8804992349b4f278c 2024-11-26T11:15:47,643 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/0db090d4ef99420787bd6768394a364d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0db090d4ef99420787bd6768394a364d 2024-11-26T11:15:47,651 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into 0db090d4ef99420787bd6768394a364d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:47,651 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:47,651 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=12, startTime=1732619747595; duration=0sec 2024-11-26T11:15:47,651 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:47,651 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:15:47,651 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:47,653 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:47,655 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:15:47,655 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:47,655 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/e28f51417c9545baaa5a3515ba0691c4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/319573b947f445aea15b0550ae111dc7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/146c9d144d7c428d8d2925209b0c9e6b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/3dc7300a00b848f9ba1c1a56a95410b5] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=46.9 K 2024-11-26T11:15:47,656 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28f51417c9545baaa5a3515ba0691c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732619744647 2024-11-26T11:15:47,656 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319573b947f445aea15b0550ae111dc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732619744726 2024-11-26T11:15:47,657 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 146c9d144d7c428d8d2925209b0c9e6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732619745058 2024-11-26T11:15:47,657 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc7300a00b848f9ba1c1a56a95410b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619745985 2024-11-26T11:15:47,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619747649, max timestamp=1732619747661Num uniques 859; 2024-11-26T11:15:47,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:47,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619747649, max timestamp=1732619747661Num uniques 859; 2024-11-26T11:15:47,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:47,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/843d3ace560f43f1afbe2cf1e0f878d9 is 50, key is test_row_0/B:col10/1732619747549/Put/seqid=0 2024-11-26T11:15:47,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619747649, max timestamp=1732619747661Num uniques 859; 2024-11-26T11:15:47,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:47,671 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:47,671 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/7a78084cba04471aaee1c07ac50d2365 is 50, key is test_row_0/C:col10/1732619745993/Put/seqid=0 2024-11-26T11:15:47,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732619747684, max timestamp=1732619747700Num uniques 1018; 2024-11-26T11:15:47,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-26T11:15:47,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732619747684, max timestamp=1732619747700Num uniques 1018; 2024-11-26T11:15:47,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-26T11:15:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742058_1234 (size=12301) 2024-11-26T11:15:47,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732619747687, max timestamp=1732619747700Num uniques 1018; 2024-11-26T11:15:47,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T11:15:47,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/843d3ace560f43f1afbe2cf1e0f878d9 2024-11-26T11:15:47,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/640e22bc3f01451fb15e048958f75f81 is 50, key is test_row_0/C:col10/1732619747549/Put/seqid=0 2024-11-26T11:15:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742059_1235 (size=12139) 2024-11-26T11:15:47,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747731, max timestamp=1732619747741Num uniques 1177; 2024-11-26T11:15:47,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-26T11:15:47,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747731, max timestamp=1732619747741Num uniques 1177; 2024-11-26T11:15:47,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-26T11:15:47,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732619747732, max timestamp=1732619747741Num uniques 1027; 2024-11-26T11:15:47,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T11:15:47,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742060_1236 (size=12301) 2024-11-26T11:15:47,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/640e22bc3f01451fb15e048958f75f81 2024-11-26T11:15:47,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3b8de98287d14df8804992349b4f278c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b8de98287d14df8804992349b4f278c 2024-11-26T11:15:47,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619747756, max timestamp=1732619747786Num uniques 1086; 2024-11-26T11:15:47,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619747756, max timestamp=1732619747786Num uniques 1086; 2024-11-26T11:15:47,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:47,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:47,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732619747777, max timestamp=1732619747786Num uniques 1036; 2024-11-26T11:15:47,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-26T11:15:47,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b8de98287d14df8804992349b4f278c, entries=150, sequenceid=337, filesize=12.0 K 2024-11-26T11:15:47,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/843d3ace560f43f1afbe2cf1e0f878d9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/843d3ace560f43f1afbe2cf1e0f878d9 2024-11-26T11:15:47,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/843d3ace560f43f1afbe2cf1e0f878d9, entries=150, sequenceid=337, filesize=12.0 K 2024-11-26T11:15:47,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/640e22bc3f01451fb15e048958f75f81 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/640e22bc3f01451fb15e048958f75f81 2024-11-26T11:15:47,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/640e22bc3f01451fb15e048958f75f81, entries=150, sequenceid=337, filesize=12.0 K 2024-11-26T11:15:47,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.28 KB/67872, currentSize=123.63 KB/126594 for de67c03f4cecf266dfeecbd54a45d9f0 in 238ms, sequenceid=337, compaction requested=false 2024-11-26T11:15:47,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:47,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=130.34 KB heapSize=369.03 KB 2024-11-26T11:15:47,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619747804, max timestamp=1732619747838Num uniques 1095; 2024-11-26T11:15:47,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T11:15:47,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619747804, max timestamp=1732619747838Num uniques 1095; 2024-11-26T11:15:47,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T11:15:47,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619747804, max timestamp=1732619747838Num uniques 1009; 2024-11-26T11:15:47,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-26T11:15:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:47,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/09d653af312e419abc34bc4e742158d3 is 50, key is test_row_0/A:col10/1732619747837/Put/seqid=0 2024-11-26T11:15:47,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742061_1237 (size=14741) 2024-11-26T11:15:47,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/09d653af312e419abc34bc4e742158d3 2024-11-26T11:15:47,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/6067188b75d345d581c9f1e4731de466 is 50, key is test_row_0/B:col10/1732619747837/Put/seqid=0 2024-11-26T11:15:47,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742062_1238 (size=12301) 2024-11-26T11:15:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619747879, max timestamp=1732619747891Num uniques 859; 2024-11-26T11:15:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:15:47,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619747879, max timestamp=1732619747891Num uniques 859; 2024-11-26T11:15:47,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:15:47,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619747879, max timestamp=1732619747891Num uniques 859; 2024-11-26T11:15:47,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:15:47,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732619747918, max timestamp=1732619747926Num uniques 1068; 2024-11-26T11:15:47,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-26T11:15:47,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732619747918, max timestamp=1732619747926Num uniques 1068; 2024-11-26T11:15:47,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-26T11:15:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732619747918, max timestamp=1732619747926Num uniques 1068; 2024-11-26T11:15:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-26T11:15:47,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619807957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:47,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:47,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619807958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619807960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:47,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:47,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:47,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:47,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747935, max timestamp=1732619747967Num uniques 1127; 2024-11-26T11:15:47,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-26T11:15:47,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747935, max timestamp=1732619747967Num uniques 1127; 2024-11-26T11:15:47,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-26T11:15:47,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619747935, max timestamp=1732619747967Num uniques 1127; 2024-11-26T11:15:47,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-26T11:15:48,030 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/739fa900a7054c0681d2c0f229bed1f3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/739fa900a7054c0681d2c0f229bed1f3 2024-11-26T11:15:48,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619748014, max timestamp=1732619748033Num uniques 1086; 2024-11-26T11:15:48,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:48,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619748014, max timestamp=1732619748033Num uniques 1086; 2024-11-26T11:15:48,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:48,039 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into 739fa900a7054c0681d2c0f229bed1f3(size=11.9 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:48,039 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:48,039 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=12, startTime=1732619747594; duration=0sec 2024-11-26T11:15:48,040 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:48,040 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:15:48,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732619748014, max timestamp=1732619748033Num uniques 1086; 2024-11-26T11:15:48,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T11:15:48,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619748068, max timestamp=1732619748100Num uniques 1095; 2024-11-26T11:15:48,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619748068, max timestamp=1732619748100Num uniques 1095; 2024-11-26T11:15:48,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T11:15:48,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T11:15:48,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732619748068, max timestamp=1732619748100Num uniques 1095; 2024-11-26T11:15:48,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T11:15:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619808125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732619748121, max timestamp=1732619748131Num uniques 1104; 2024-11-26T11:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732619748121, max timestamp=1732619748131Num uniques 1104; 2024-11-26T11:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732619748121, max timestamp=1732619748131Num uniques 1104; 2024-11-26T11:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-26T11:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-26T11:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-26T11:15:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,151 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/7a78084cba04471aaee1c07ac50d2365 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7a78084cba04471aaee1c07ac50d2365 2024-11-26T11:15:48,157 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into 7a78084cba04471aaee1c07ac50d2365(size=11.9 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:48,157 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:48,157 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=12, startTime=1732619747596; duration=0sec 2024-11-26T11:15:48,157 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:48,157 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:15:48,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732619748147, max timestamp=1732619748165Num uniques 1113; 2024-11-26T11:15:48,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-11-26T11:15:48,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732619748157, max timestamp=1732619748165Num uniques 1113; 2024-11-26T11:15:48,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=6870, suffix heap size=143720,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=150 2024-11-26T11:15:48,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732619748147, max timestamp=1732619748165Num uniques 1113; 2024-11-26T11:15:48,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-11-26T11:15:48,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619808204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732619748200, max timestamp=1732619748208Num uniques 1172; 2024-11-26T11:15:48,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=7872, suffix heap size=149856,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=172 2024-11-26T11:15:48,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732619748200, max timestamp=1732619748208Num uniques 1009; 2024-11-26T11:15:48,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7872, suffix heap size=132904,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=172 2024-11-26T11:15:48,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732619748200, max timestamp=1732619748208Num uniques 1172; 2024-11-26T11:15:48,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=7872, suffix heap size=149856,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=172 2024-11-26T11:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732619748213, max timestamp=1732619748258Num uniques 1031; 2024-11-26T11:15:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-26T11:15:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732619748213, max timestamp=1732619748258Num uniques 1031; 2024-11-26T11:15:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732619748213, max timestamp=1732619748258Num uniques 1031; 2024-11-26T11:15:48,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-26T11:15:48,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-26T11:15:48,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/6067188b75d345d581c9f1e4731de466 2024-11-26T11:15:48,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732619748287, max timestamp=1732619748297Num uniques 1090; 2024-11-26T11:15:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-26T11:15:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732619748287, max timestamp=1732619748297Num uniques 1090; 2024-11-26T11:15:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-26T11:15:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732619748287, max timestamp=1732619748297Num uniques 1090; 2024-11-26T11:15:48,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/742a330d6694460dabf4406717a0ce95 is 50, key is test_row_0/C:col10/1732619747837/Put/seqid=0 2024-11-26T11:15:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-26T11:15:48,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742063_1239 (size=12301) 2024-11-26T11:15:48,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/742a330d6694460dabf4406717a0ce95 2024-11-26T11:15:48,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/09d653af312e419abc34bc4e742158d3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/09d653af312e419abc34bc4e742158d3 2024-11-26T11:15:48,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732619748328, max timestamp=1732619748341Num uniques 1049; 2024-11-26T11:15:48,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-26T11:15:48,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732619748328, max timestamp=1732619748341Num uniques 1049; 2024-11-26T11:15:48,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-26T11:15:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732619748328, max timestamp=1732619748341Num uniques 1049; 2024-11-26T11:15:48,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-26T11:15:48,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/09d653af312e419abc34bc4e742158d3, entries=200, sequenceid=427, filesize=14.4 K 2024-11-26T11:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/6067188b75d345d581c9f1e4731de466 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/6067188b75d345d581c9f1e4731de466 2024-11-26T11:15:48,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/6067188b75d345d581c9f1e4731de466, entries=150, sequenceid=427, filesize=12.0 K 2024-11-26T11:15:48,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/742a330d6694460dabf4406717a0ce95 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/742a330d6694460dabf4406717a0ce95 2024-11-26T11:15:48,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/742a330d6694460dabf4406717a0ce95, entries=150, sequenceid=427, filesize=12.0 K 2024-11-26T11:15:48,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~78.05 KB/79920, currentSize=93.93 KB/96180 for de67c03f4cecf266dfeecbd54a45d9f0 in 525ms, sequenceid=427, compaction requested=true 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:48,364 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:48,364 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:48,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=100.63 KB heapSize=276.94 KB 2024-11-26T11:15:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:48,365 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36741 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:48,365 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:15:48,365 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:48,366 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0db090d4ef99420787bd6768394a364d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/843d3ace560f43f1afbe2cf1e0f878d9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/6067188b75d345d581c9f1e4731de466] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=35.9 K 2024-11-26T11:15:48,366 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:48,366 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:15:48,366 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:48,366 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/739fa900a7054c0681d2c0f229bed1f3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b8de98287d14df8804992349b4f278c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/09d653af312e419abc34bc4e742158d3] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=38.3 K 2024-11-26T11:15:48,366 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db090d4ef99420787bd6768394a364d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619745985 2024-11-26T11:15:48,369 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739fa900a7054c0681d2c0f229bed1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619745985 2024-11-26T11:15:48,369 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 843d3ace560f43f1afbe2cf1e0f878d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732619747547 2024-11-26T11:15:48,371 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8de98287d14df8804992349b4f278c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732619747547 2024-11-26T11:15:48,371 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6067188b75d345d581c9f1e4731de466, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732619747836 2024-11-26T11:15:48,371 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d653af312e419abc34bc4e742158d3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732619747804 2024-11-26T11:15:48,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/adbf7ab0f9ff421a865d8a5e195a23b7 is 50, key is test_row_0/A:col10/1732619748357/Put/seqid=0 2024-11-26T11:15:48,390 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:48,391 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/47dea3d986b94b36b4f27d0bac6edb6e is 50, key is test_row_0/B:col10/1732619747837/Put/seqid=0 2024-11-26T11:15:48,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619808386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619808387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,398 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:48,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619808392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,399 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/538dc808fa48437c86910d600e67a8dc is 50, key is test_row_0/A:col10/1732619747837/Put/seqid=0 2024-11-26T11:15:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619808395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619808395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742064_1240 (size=12301) 2024-11-26T11:15:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742065_1241 (size=12541) 2024-11-26T11:15:48,442 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/47dea3d986b94b36b4f27d0bac6edb6e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/47dea3d986b94b36b4f27d0bac6edb6e 2024-11-26T11:15:48,451 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into 47dea3d986b94b36b4f27d0bac6edb6e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:48,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:48,451 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619748364; duration=0sec 2024-11-26T11:15:48,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:48,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:15:48,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:48,452 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36741 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:48,452 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:15:48,452 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:48,452 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7a78084cba04471aaee1c07ac50d2365, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/640e22bc3f01451fb15e048958f75f81, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/742a330d6694460dabf4406717a0ce95] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=35.9 K 2024-11-26T11:15:48,452 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a78084cba04471aaee1c07ac50d2365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732619745985 2024-11-26T11:15:48,453 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 640e22bc3f01451fb15e048958f75f81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732619747547 2024-11-26T11:15:48,453 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 742a330d6694460dabf4406717a0ce95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732619747836 2024-11-26T11:15:48,463 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:48,464 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/338d1f2aa3e4414081d924709155c885 is 50, key is test_row_0/C:col10/1732619747837/Put/seqid=0 2024-11-26T11:15:48,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742066_1242 (size=12541) 2024-11-26T11:15:48,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742067_1243 (size=12541) 2024-11-26T11:15:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619808503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619808503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619808503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619808513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619808513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619808713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619808713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619808713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619808722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619808723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T11:15:48,813 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:48,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-26T11:15:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T11:15:48,815 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:48,817 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:48,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:48,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/adbf7ab0f9ff421a865d8a5e195a23b7 2024-11-26T11:15:48,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/8321a6a2888344e49468dd583a9e8d7a is 50, key is test_row_0/B:col10/1732619748357/Put/seqid=0 2024-11-26T11:15:48,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742068_1244 (size=12301) 2024-11-26T11:15:48,876 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/538dc808fa48437c86910d600e67a8dc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/538dc808fa48437c86910d600e67a8dc 2024-11-26T11:15:48,880 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into 538dc808fa48437c86910d600e67a8dc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:48,880 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:48,880 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619748364; duration=0sec 2024-11-26T11:15:48,881 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:48,881 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:15:48,893 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/338d1f2aa3e4414081d924709155c885 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/338d1f2aa3e4414081d924709155c885 2024-11-26T11:15:48,899 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into 338d1f2aa3e4414081d924709155c885(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:48,899 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:48,899 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619748364; duration=0sec 2024-11-26T11:15:48,899 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:48,899 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T11:15:48,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T11:15:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:48,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619809023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619809023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619809025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619809033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619809033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T11:15:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T11:15:49,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/8321a6a2888344e49468dd583a9e8d7a 2024-11-26T11:15:49,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/6c0ce872ed1b456a837de06e48910289 is 50, key is test_row_0/C:col10/1732619748357/Put/seqid=0 2024-11-26T11:15:49,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742069_1245 (size=12301) 2024-11-26T11:15:49,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T11:15:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T11:15:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T11:15:49,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619809533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619809533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619809543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619809543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619809552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:49,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T11:15:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:49,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/6c0ce872ed1b456a837de06e48910289 2024-11-26T11:15:49,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/adbf7ab0f9ff421a865d8a5e195a23b7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/adbf7ab0f9ff421a865d8a5e195a23b7 2024-11-26T11:15:49,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/adbf7ab0f9ff421a865d8a5e195a23b7, entries=150, sequenceid=631, filesize=12.0 K 2024-11-26T11:15:49,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/8321a6a2888344e49468dd583a9e8d7a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/8321a6a2888344e49468dd583a9e8d7a 2024-11-26T11:15:49,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/8321a6a2888344e49468dd583a9e8d7a, entries=150, sequenceid=631, filesize=12.0 K 2024-11-26T11:15:49,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/6c0ce872ed1b456a837de06e48910289 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6c0ce872ed1b456a837de06e48910289 2024-11-26T11:15:49,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6c0ce872ed1b456a837de06e48910289, entries=150, sequenceid=631, filesize=12.0 K 2024-11-26T11:15:49,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~276.89 KB/283536, currentSize=87.22 KB/89310 for de67c03f4cecf266dfeecbd54a45d9f0 in 1316ms, sequenceid=631, compaction requested=false 2024-11-26T11:15:49,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:49,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T11:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:49,743 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:49,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b326ba3e83804affbba186c7873a5383 is 50, key is test_row_0/A:col10/1732619748392/Put/seqid=0 2024-11-26T11:15:49,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742070_1246 (size=12301) 2024-11-26T11:15:49,766 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b326ba3e83804affbba186c7873a5383 2024-11-26T11:15:49,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d65f904839f44e42888334a7acb0060b is 50, key is test_row_0/B:col10/1732619748392/Put/seqid=0 2024-11-26T11:15:49,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742071_1247 (size=12301) 2024-11-26T11:15:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T11:15:50,188 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d65f904839f44e42888334a7acb0060b 2024-11-26T11:15:50,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b75687188fc44925bc18c5ed06b3a214 is 50, key is test_row_0/C:col10/1732619748392/Put/seqid=0 2024-11-26T11:15:50,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742072_1248 (size=12301) 2024-11-26T11:15:50,215 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b75687188fc44925bc18c5ed06b3a214 2024-11-26T11:15:50,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b326ba3e83804affbba186c7873a5383 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b326ba3e83804affbba186c7873a5383 2024-11-26T11:15:50,228 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b326ba3e83804affbba186c7873a5383, entries=150, sequenceid=650, filesize=12.0 K 2024-11-26T11:15:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d65f904839f44e42888334a7acb0060b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d65f904839f44e42888334a7acb0060b 2024-11-26T11:15:50,237 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d65f904839f44e42888334a7acb0060b, entries=150, sequenceid=650, filesize=12.0 K 2024-11-26T11:15:50,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b75687188fc44925bc18c5ed06b3a214 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b75687188fc44925bc18c5ed06b3a214 2024-11-26T11:15:50,245 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b75687188fc44925bc18c5ed06b3a214, entries=150, sequenceid=650, filesize=12.0 K 2024-11-26T11:15:50,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for de67c03f4cecf266dfeecbd54a45d9f0 in 503ms, sequenceid=650, compaction requested=true 2024-11-26T11:15:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T11:15:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T11:15:50,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T11:15:50,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-26T11:15:50,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4390 sec 2024-11-26T11:15:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:50,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:50,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/537d8dff68c541c68ac572584db05e5e is 50, key is test_row_0/A:col10/1732619750547/Put/seqid=0 2024-11-26T11:15:50,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742073_1249 (size=14741) 2024-11-26T11:15:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619810580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732619750574, max timestamp=1732619750587Num uniques 859; 2024-11-26T11:15:50,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T11:15:50,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732619750574, max timestamp=1732619750587Num uniques 859; 2024-11-26T11:15:50,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T11:15:50,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732619750574, max timestamp=1732619750587Num uniques 859; 2024-11-26T11:15:50,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T11:15:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619810613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619810618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619810619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619810620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619810692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619810723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619810733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619810733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619810733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619810903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619810933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619810943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619810943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619810943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T11:15:50,963 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:50,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-26T11:15:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T11:15:50,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:50,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:50,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:50,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/537d8dff68c541c68ac572584db05e5e 2024-11-26T11:15:50,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/87606aef66624fe4b6d94c859138edc6 is 50, key is test_row_0/B:col10/1732619750547/Put/seqid=0 2024-11-26T11:15:50,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742074_1250 (size=12301) 2024-11-26T11:15:50,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/87606aef66624fe4b6d94c859138edc6 2024-11-26T11:15:51,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d76fbc1902694a28a8701f0de508bafd is 50, key is test_row_0/C:col10/1732619750547/Put/seqid=0 2024-11-26T11:15:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742075_1251 (size=12301) 2024-11-26T11:15:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T11:15:51,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T11:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:51,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619811212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619811243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619811253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619811253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619811253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T11:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T11:15:51,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d76fbc1902694a28a8701f0de508bafd 2024-11-26T11:15:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/537d8dff68c541c68ac572584db05e5e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/537d8dff68c541c68ac572584db05e5e 2024-11-26T11:15:51,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/537d8dff68c541c68ac572584db05e5e, entries=200, sequenceid=661, filesize=14.4 K 2024-11-26T11:15:51,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/87606aef66624fe4b6d94c859138edc6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/87606aef66624fe4b6d94c859138edc6 2024-11-26T11:15:51,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T11:15:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:51,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/87606aef66624fe4b6d94c859138edc6, entries=150, sequenceid=661, filesize=12.0 K 2024-11-26T11:15:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d76fbc1902694a28a8701f0de508bafd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d76fbc1902694a28a8701f0de508bafd 2024-11-26T11:15:51,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d76fbc1902694a28a8701f0de508bafd, entries=150, sequenceid=661, filesize=12.0 K 2024-11-26T11:15:51,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for de67c03f4cecf266dfeecbd54a45d9f0 in 881ms, sequenceid=661, compaction requested=true 2024-11-26T11:15:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:51,438 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:51,438 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:51,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:51,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:51,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:51,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:51,440 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51884 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:51,440 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:51,440 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:15:51,440 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:15:51,440 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,440 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,440 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/538dc808fa48437c86910d600e67a8dc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/adbf7ab0f9ff421a865d8a5e195a23b7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b326ba3e83804affbba186c7873a5383, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/537d8dff68c541c68ac572584db05e5e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=50.7 K 2024-11-26T11:15:51,440 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/47dea3d986b94b36b4f27d0bac6edb6e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/8321a6a2888344e49468dd583a9e8d7a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d65f904839f44e42888334a7acb0060b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/87606aef66624fe4b6d94c859138edc6] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=48.3 K 2024-11-26T11:15:51,440 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 47dea3d986b94b36b4f27d0bac6edb6e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732619747836 2024-11-26T11:15:51,440 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 538dc808fa48437c86910d600e67a8dc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732619747836 2024-11-26T11:15:51,441 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 8321a6a2888344e49468dd583a9e8d7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1732619748357 2024-11-26T11:15:51,441 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting adbf7ab0f9ff421a865d8a5e195a23b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1732619748357 2024-11-26T11:15:51,441 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b326ba3e83804affbba186c7873a5383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732619748390 2024-11-26T11:15:51,441 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d65f904839f44e42888334a7acb0060b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732619748390 2024-11-26T11:15:51,442 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 87606aef66624fe4b6d94c859138edc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1732619750547 2024-11-26T11:15:51,442 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537d8dff68c541c68ac572584db05e5e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1732619750547 2024-11-26T11:15:51,451 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:51,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/f293ae4c0cff4bcd99d3c10456dc41ec is 50, key is test_row_0/B:col10/1732619750547/Put/seqid=0 2024-11-26T11:15:51,451 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:51,452 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3b1cb81fa094473d953fa85eacadad71 is 50, key is test_row_0/A:col10/1732619750547/Put/seqid=0 2024-11-26T11:15:51,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742076_1252 (size=12677) 2024-11-26T11:15:51,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742077_1253 (size=12677) 2024-11-26T11:15:51,468 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/f293ae4c0cff4bcd99d3c10456dc41ec as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/f293ae4c0cff4bcd99d3c10456dc41ec 2024-11-26T11:15:51,470 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3b1cb81fa094473d953fa85eacadad71 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b1cb81fa094473d953fa85eacadad71 2024-11-26T11:15:51,473 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into f293ae4c0cff4bcd99d3c10456dc41ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:51,473 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:51,473 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=12, startTime=1732619751438; duration=0sec 2024-11-26T11:15:51,473 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:51,473 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:15:51,473 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:15:51,475 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:15:51,475 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:15:51,475 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,475 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/338d1f2aa3e4414081d924709155c885, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6c0ce872ed1b456a837de06e48910289, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b75687188fc44925bc18c5ed06b3a214, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d76fbc1902694a28a8701f0de508bafd] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=48.3 K 2024-11-26T11:15:51,475 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 338d1f2aa3e4414081d924709155c885, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732619747836 2024-11-26T11:15:51,476 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0ce872ed1b456a837de06e48910289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1732619748357 2024-11-26T11:15:51,476 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b75687188fc44925bc18c5ed06b3a214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732619748390 2024-11-26T11:15:51,476 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d76fbc1902694a28a8701f0de508bafd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1732619750547 2024-11-26T11:15:51,476 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into 3b1cb81fa094473d953fa85eacadad71(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:51,476 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:51,476 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=12, startTime=1732619751438; duration=0sec 2024-11-26T11:15:51,476 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:51,476 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:15:51,484 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:51,485 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b021f9895e61435484c50ba2de5d8d46 is 50, key is test_row_0/C:col10/1732619750547/Put/seqid=0 2024-11-26T11:15:51,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742078_1254 (size=12677) 2024-11-26T11:15:51,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T11:15:51,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:51,576 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-26T11:15:51,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:51,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:51,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:51,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b889911747f34b37a6db372b1754a89b is 50, key is test_row_0/A:col10/1732619750609/Put/seqid=0 2024-11-26T11:15:51,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742079_1255 (size=12301) 2024-11-26T11:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T11:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:51,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:51,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619811758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619811762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619811763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619811763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619811772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619811873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619811873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:51,896 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b021f9895e61435484c50ba2de5d8d46 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b021f9895e61435484c50ba2de5d8d46 2024-11-26T11:15:51,906 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into b021f9895e61435484c50ba2de5d8d46(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:51,906 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:51,906 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=12, startTime=1732619751439; duration=0sec 2024-11-26T11:15:51,906 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:51,907 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:15:51,989 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b889911747f34b37a6db372b1754a89b 2024-11-26T11:15:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/befd55f6efdf485db31d3819012c9bf8 is 50, key is test_row_0/B:col10/1732619750609/Put/seqid=0 2024-11-26T11:15:52,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742080_1256 (size=12301) 2024-11-26T11:15:52,057 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/befd55f6efdf485db31d3819012c9bf8 2024-11-26T11:15:52,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/cd0fdf2a2665478f891ac04a614c8e37 is 50, key is test_row_0/C:col10/1732619750609/Put/seqid=0 2024-11-26T11:15:52,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742081_1257 (size=12301) 2024-11-26T11:15:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619812083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619812083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T11:15:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619812392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619812392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,482 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/cd0fdf2a2665478f891ac04a614c8e37 2024-11-26T11:15:52,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b889911747f34b37a6db372b1754a89b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b889911747f34b37a6db372b1754a89b 2024-11-26T11:15:52,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b889911747f34b37a6db372b1754a89b, entries=150, sequenceid=698, filesize=12.0 K 2024-11-26T11:15:52,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/befd55f6efdf485db31d3819012c9bf8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/befd55f6efdf485db31d3819012c9bf8 2024-11-26T11:15:52,493 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/befd55f6efdf485db31d3819012c9bf8, entries=150, sequenceid=698, filesize=12.0 K 2024-11-26T11:15:52,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/cd0fdf2a2665478f891ac04a614c8e37 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/cd0fdf2a2665478f891ac04a614c8e37 2024-11-26T11:15:52,498 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/cd0fdf2a2665478f891ac04a614c8e37, entries=150, sequenceid=698, filesize=12.0 K 2024-11-26T11:15:52,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for de67c03f4cecf266dfeecbd54a45d9f0 in 923ms, sequenceid=698, compaction requested=false 2024-11-26T11:15:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T11:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T11:15:52,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T11:15:52,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-11-26T11:15:52,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.5390 sec 2024-11-26T11:15:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:52,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:15:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:52,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/721f0f069ac4497f8397ab31b5d28f2a is 50, key is test_row_0/A:col10/1732619751760/Put/seqid=0 2024-11-26T11:15:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742082_1258 (size=14741) 2024-11-26T11:15:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619812821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619812826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619812829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619812902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619812903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619812933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619812933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:52,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619812942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T11:15:53,113 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:53,113 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-26T11:15:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T11:15:53,115 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:53,116 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:53,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619813143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619813143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619813152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/721f0f069ac4497f8397ab31b5d28f2a 2024-11-26T11:15:53,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d0a9f63cd7eb472cb170049ec18332bf is 50, key is test_row_0/B:col10/1732619751760/Put/seqid=0 2024-11-26T11:15:53,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742083_1259 (size=12301) 2024-11-26T11:15:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T11:15:53,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T11:15:53,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619813453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619813453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619813463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d0a9f63cd7eb472cb170049ec18332bf 2024-11-26T11:15:53,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b186319d41664fe482e09042d3a3737a is 50, key is test_row_0/C:col10/1732619751760/Put/seqid=0 2024-11-26T11:15:53,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742084_1260 (size=12301) 2024-11-26T11:15:53,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T11:15:53,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:53,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:53,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:53,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619813913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,917 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619813923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,925 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619813963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619813963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619813972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:54,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b186319d41664fe482e09042d3a3737a 2024-11-26T11:15:54,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:54,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:54,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:54,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:54,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/721f0f069ac4497f8397ab31b5d28f2a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/721f0f069ac4497f8397ab31b5d28f2a 2024-11-26T11:15:54,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/721f0f069ac4497f8397ab31b5d28f2a, entries=200, sequenceid=712, filesize=14.4 K 2024-11-26T11:15:54,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d0a9f63cd7eb472cb170049ec18332bf as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0a9f63cd7eb472cb170049ec18332bf 2024-11-26T11:15:54,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0a9f63cd7eb472cb170049ec18332bf, entries=150, sequenceid=712, filesize=12.0 K 2024-11-26T11:15:54,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/b186319d41664fe482e09042d3a3737a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b186319d41664fe482e09042d3a3737a 2024-11-26T11:15:54,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b186319d41664fe482e09042d3a3737a, entries=150, sequenceid=712, filesize=12.0 K 2024-11-26T11:15:54,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for de67c03f4cecf266dfeecbd54a45d9f0 in 1271ms, sequenceid=712, compaction requested=true 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:54,058 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:54,058 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:54,059 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:54,059 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:54,059 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:15:54,059 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:15:54,059 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,059 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,059 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/f293ae4c0cff4bcd99d3c10456dc41ec, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/befd55f6efdf485db31d3819012c9bf8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0a9f63cd7eb472cb170049ec18332bf] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.4 K 2024-11-26T11:15:54,059 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b1cb81fa094473d953fa85eacadad71, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b889911747f34b37a6db372b1754a89b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/721f0f069ac4497f8397ab31b5d28f2a] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=38.8 K 2024-11-26T11:15:54,060 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f293ae4c0cff4bcd99d3c10456dc41ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1732619750547 2024-11-26T11:15:54,060 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1cb81fa094473d953fa85eacadad71, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1732619750547 2024-11-26T11:15:54,061 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b889911747f34b37a6db372b1754a89b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1732619750609 2024-11-26T11:15:54,061 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting befd55f6efdf485db31d3819012c9bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1732619750609 2024-11-26T11:15:54,061 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721f0f069ac4497f8397ab31b5d28f2a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1732619751744 2024-11-26T11:15:54,061 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a9f63cd7eb472cb170049ec18332bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1732619751744 2024-11-26T11:15:54,072 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:54,072 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/ed508724fb8d40fc8400d721e403cd7a is 50, key is test_row_0/B:col10/1732619751760/Put/seqid=0 2024-11-26T11:15:54,073 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:54,074 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/0b1c43541a754f46b4d3c71c40633460 is 50, key is test_row_0/A:col10/1732619751760/Put/seqid=0 2024-11-26T11:15:54,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742085_1261 (size=12779) 2024-11-26T11:15:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742086_1262 (size=12779) 2024-11-26T11:15:54,108 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/0b1c43541a754f46b4d3c71c40633460 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/0b1c43541a754f46b4d3c71c40633460 2024-11-26T11:15:54,114 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into 0b1c43541a754f46b4d3c71c40633460(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:54,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:54,114 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619754058; duration=0sec 2024-11-26T11:15:54,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:54,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:15:54,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:54,115 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:54,115 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:15:54,116 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,116 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b021f9895e61435484c50ba2de5d8d46, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/cd0fdf2a2665478f891ac04a614c8e37, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b186319d41664fe482e09042d3a3737a] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.4 K 2024-11-26T11:15:54,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b021f9895e61435484c50ba2de5d8d46, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=661, earliestPutTs=1732619750547 2024-11-26T11:15:54,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd0fdf2a2665478f891ac04a614c8e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1732619750609 2024-11-26T11:15:54,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b186319d41664fe482e09042d3a3737a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1732619751744 2024-11-26T11:15:54,126 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:54,127 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/c06f41ea99574e8dba9f35d4be2b69a3 is 50, key is test_row_0/C:col10/1732619751760/Put/seqid=0 2024-11-26T11:15:54,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742087_1263 (size=12779) 2024-11-26T11:15:54,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T11:15:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,187 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:15:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:54,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/8509b84a7f214449b1643c3c686acb77 is 50, key is test_row_0/A:col10/1732619752797/Put/seqid=0 2024-11-26T11:15:54,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742088_1264 (size=12301) 2024-11-26T11:15:54,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=732 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/8509b84a7f214449b1643c3c686acb77 2024-11-26T11:15:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/a37b6aa67bae45cc9f8d722749a4d4f3 is 50, key is test_row_0/B:col10/1732619752797/Put/seqid=0 2024-11-26T11:15:54,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742089_1265 (size=12301) 2024-11-26T11:15:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T11:15:54,495 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/ed508724fb8d40fc8400d721e403cd7a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ed508724fb8d40fc8400d721e403cd7a 2024-11-26T11:15:54,499 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into ed508724fb8d40fc8400d721e403cd7a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:54,499 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:54,499 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619754058; duration=0sec 2024-11-26T11:15:54,499 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:54,499 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:15:54,550 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/c06f41ea99574e8dba9f35d4be2b69a3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/c06f41ea99574e8dba9f35d4be2b69a3 2024-11-26T11:15:54,559 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into c06f41ea99574e8dba9f35d4be2b69a3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:54,560 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:54,560 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619754058; duration=0sec 2024-11-26T11:15:54,560 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:54,560 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:15:54,620 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=732 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/a37b6aa67bae45cc9f8d722749a4d4f3 2024-11-26T11:15:54,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/85b0e85f82f14bbba983db0aa7e22191 is 50, key is test_row_0/C:col10/1732619752797/Put/seqid=0 2024-11-26T11:15:54,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742090_1266 (size=12301) 2024-11-26T11:15:54,651 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=732 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/85b0e85f82f14bbba983db0aa7e22191 2024-11-26T11:15:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/8509b84a7f214449b1643c3c686acb77 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/8509b84a7f214449b1643c3c686acb77 2024-11-26T11:15:54,664 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/8509b84a7f214449b1643c3c686acb77, entries=150, sequenceid=732, filesize=12.0 K 2024-11-26T11:15:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/a37b6aa67bae45cc9f8d722749a4d4f3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/a37b6aa67bae45cc9f8d722749a4d4f3 2024-11-26T11:15:54,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/a37b6aa67bae45cc9f8d722749a4d4f3, entries=150, sequenceid=732, filesize=12.0 K 2024-11-26T11:15:54,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/85b0e85f82f14bbba983db0aa7e22191 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/85b0e85f82f14bbba983db0aa7e22191 2024-11-26T11:15:54,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/85b0e85f82f14bbba983db0aa7e22191, entries=150, sequenceid=732, filesize=12.0 K 2024-11-26T11:15:54,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for de67c03f4cecf266dfeecbd54a45d9f0 in 494ms, sequenceid=732, compaction requested=false 2024-11-26T11:15:54,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T11:15:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T11:15:54,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T11:15:54,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-26T11:15:54,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5740 sec 2024-11-26T11:15:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:54,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:15:54,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:54,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:54,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:54,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:54,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:54,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:54,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/ee22068760be4254b03fef9354ac6f85 is 50, key is test_row_0/A:col10/1732619754981/Put/seqid=0 2024-11-26T11:15:55,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742091_1267 (size=17181) 2024-11-26T11:15:55,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619815010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619815011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619755004, max timestamp=1732619755015Num uniques 859; 2024-11-26T11:15:55,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:55,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619755004, max timestamp=1732619755015Num uniques 859; 2024-11-26T11:15:55,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:55,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619755004, max timestamp=1732619755015Num uniques 859; 2024-11-26T11:15:55,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619815086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619815123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619815123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619815194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T11:15:55,263 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:55,263 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-26T11:15:55,265 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T11:15:55,265 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:55,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619815332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619815332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T11:15:55,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=745 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/ee22068760be4254b03fef9354ac6f85 2024-11-26T11:15:55,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619815403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/b600e440c6bc4f8c9c43b2acdc85a0a0 is 50, key is test_row_0/B:col10/1732619754981/Put/seqid=0 2024-11-26T11:15:55,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:55,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:55,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742092_1268 (size=12301) 2024-11-26T11:15:55,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:55,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:55,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T11:15:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619815642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619815642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619815713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=745 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/b600e440c6bc4f8c9c43b2acdc85a0a0 2024-11-26T11:15:55,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/ea1b0296f77043939ca4ba7b9394287e is 50, key is test_row_0/C:col10/1732619754981/Put/seqid=0 2024-11-26T11:15:55,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742093_1269 (size=12301) 2024-11-26T11:15:55,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:55,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T11:15:55,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619815923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,926 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619815942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:55,945 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:56,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:56,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:56,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:56,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:56,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619816153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:56,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619816153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:56,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:56,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619816223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:56,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=745 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/ea1b0296f77043939ca4ba7b9394287e 2024-11-26T11:15:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/ee22068760be4254b03fef9354ac6f85 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/ee22068760be4254b03fef9354ac6f85 2024-11-26T11:15:56,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/ee22068760be4254b03fef9354ac6f85, entries=250, sequenceid=745, filesize=16.8 K 2024-11-26T11:15:56,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/b600e440c6bc4f8c9c43b2acdc85a0a0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b600e440c6bc4f8c9c43b2acdc85a0a0 2024-11-26T11:15:56,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b600e440c6bc4f8c9c43b2acdc85a0a0, entries=150, sequenceid=745, filesize=12.0 K 2024-11-26T11:15:56,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/ea1b0296f77043939ca4ba7b9394287e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ea1b0296f77043939ca4ba7b9394287e 2024-11-26T11:15:56,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ea1b0296f77043939ca4ba7b9394287e, entries=150, sequenceid=745, filesize=12.0 K 2024-11-26T11:15:56,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for de67c03f4cecf266dfeecbd54a45d9f0 in 1293ms, sequenceid=745, compaction requested=true 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:56,277 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:56,277 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:56,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:56,278 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:56,278 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42261 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:56,278 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:15:56,278 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:15:56,278 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,278 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,278 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/0b1c43541a754f46b4d3c71c40633460, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/8509b84a7f214449b1643c3c686acb77, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/ee22068760be4254b03fef9354ac6f85] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=41.3 K 2024-11-26T11:15:56,278 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ed508724fb8d40fc8400d721e403cd7a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/a37b6aa67bae45cc9f8d722749a4d4f3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b600e440c6bc4f8c9c43b2acdc85a0a0] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.5 K 2024-11-26T11:15:56,278 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b1c43541a754f46b4d3c71c40633460, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1732619751744 2024-11-26T11:15:56,278 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ed508724fb8d40fc8400d721e403cd7a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1732619751744 2024-11-26T11:15:56,279 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8509b84a7f214449b1643c3c686acb77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=732, earliestPutTs=1732619752797 2024-11-26T11:15:56,279 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a37b6aa67bae45cc9f8d722749a4d4f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=732, earliestPutTs=1732619752797 2024-11-26T11:15:56,279 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee22068760be4254b03fef9354ac6f85, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=745, earliestPutTs=1732619754979 2024-11-26T11:15:56,279 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b600e440c6bc4f8c9c43b2acdc85a0a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=745, earliestPutTs=1732619754979 2024-11-26T11:15:56,286 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:56,287 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/b81064cf436844cebacfcfab0a891461 is 50, key is test_row_0/B:col10/1732619754981/Put/seqid=0 2024-11-26T11:15:56,289 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:56,289 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/47efedaa019a4b83a06c784aede0caeb is 50, key is test_row_0/A:col10/1732619754981/Put/seqid=0 2024-11-26T11:15:56,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742094_1270 (size=12881) 2024-11-26T11:15:56,302 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/b81064cf436844cebacfcfab0a891461 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b81064cf436844cebacfcfab0a891461 2024-11-26T11:15:56,315 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into b81064cf436844cebacfcfab0a891461(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:56,315 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:56,315 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619756277; duration=0sec 2024-11-26T11:15:56,316 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:56,316 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:15:56,316 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:56,318 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:56,318 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:15:56,318 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,318 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/c06f41ea99574e8dba9f35d4be2b69a3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/85b0e85f82f14bbba983db0aa7e22191, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ea1b0296f77043939ca4ba7b9394287e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.5 K 2024-11-26T11:15:56,318 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c06f41ea99574e8dba9f35d4be2b69a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1732619751744 2024-11-26T11:15:56,319 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 85b0e85f82f14bbba983db0aa7e22191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=732, earliestPutTs=1732619752797 2024-11-26T11:15:56,320 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ea1b0296f77043939ca4ba7b9394287e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=745, earliestPutTs=1732619754979 2024-11-26T11:15:56,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742095_1271 (size=12881) 2024-11-26T11:15:56,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T11:15:56,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:56,336 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-26T11:15:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:15:56,342 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:56,343 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/a2769fb6b9104d9eaf6af7c3f54deea9 is 50, key is test_row_0/C:col10/1732619754981/Put/seqid=0 2024-11-26T11:15:56,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/d9a0919e0b1e4cc28da8cd985e9c964c is 50, key is test_row_0/A:col10/1732619755085/Put/seqid=0 2024-11-26T11:15:56,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742096_1272 (size=12881) 2024-11-26T11:15:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742097_1273 (size=12301) 2024-11-26T11:15:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T11:15:56,731 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/47efedaa019a4b83a06c784aede0caeb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/47efedaa019a4b83a06c784aede0caeb 2024-11-26T11:15:56,736 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into 47efedaa019a4b83a06c784aede0caeb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:56,736 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:56,736 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619756277; duration=0sec 2024-11-26T11:15:56,736 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:56,736 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:15:56,779 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/a2769fb6b9104d9eaf6af7c3f54deea9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/a2769fb6b9104d9eaf6af7c3f54deea9 2024-11-26T11:15:56,785 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into a2769fb6b9104d9eaf6af7c3f54deea9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:56,785 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:56,785 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619756277; duration=0sec 2024-11-26T11:15:56,786 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:56,786 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:15:56,786 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=782 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/d9a0919e0b1e4cc28da8cd985e9c964c 2024-11-26T11:15:56,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/9440d144fd3647ddaecf2328e7c237be is 50, key is test_row_0/B:col10/1732619755085/Put/seqid=0 2024-11-26T11:15:56,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742098_1274 (size=12301) 2024-11-26T11:15:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:57,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619817191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619817196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,204 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=782 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/9440d144fd3647ddaecf2328e7c237be 2024-11-26T11:15:57,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/87ffe6a0daff41c1bd3bbaab4970d2b4 is 50, key is test_row_0/C:col10/1732619755085/Put/seqid=0 2024-11-26T11:15:57,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619817242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,248 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742099_1275 (size=12301) 2024-11-26T11:15:57,279 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=782 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/87ffe6a0daff41c1bd3bbaab4970d2b4 2024-11-26T11:15:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/d9a0919e0b1e4cc28da8cd985e9c964c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/d9a0919e0b1e4cc28da8cd985e9c964c 2024-11-26T11:15:57,305 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/d9a0919e0b1e4cc28da8cd985e9c964c, entries=150, sequenceid=782, filesize=12.0 K 2024-11-26T11:15:57,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619817304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619817304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/9440d144fd3647ddaecf2328e7c237be as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9440d144fd3647ddaecf2328e7c237be 2024-11-26T11:15:57,324 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9440d144fd3647ddaecf2328e7c237be, entries=150, sequenceid=782, filesize=12.0 K 2024-11-26T11:15:57,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/87ffe6a0daff41c1bd3bbaab4970d2b4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/87ffe6a0daff41c1bd3bbaab4970d2b4 2024-11-26T11:15:57,348 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/87ffe6a0daff41c1bd3bbaab4970d2b4, entries=150, sequenceid=782, filesize=12.0 K 2024-11-26T11:15:57,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for de67c03f4cecf266dfeecbd54a45d9f0 in 1014ms, sequenceid=782, compaction requested=false 2024-11-26T11:15:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T11:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T11:15:57,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T11:15:57,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-26T11:15:57,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0930 sec 2024-11-26T11:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T11:15:57,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:15:57,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-26T11:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:15:57,417 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:15:57,418 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:15:57,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:57,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:15:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:57,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/e0cc7009fdb941849728d053c9fb04c5 is 50, key is test_row_0/A:col10/1732619757514/Put/seqid=0 2024-11-26T11:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:15:57,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742100_1276 (size=14741) 2024-11-26T11:15:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619817553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619817557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619817662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619817672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:15:57,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:57,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619817873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619817882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:57,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/e0cc7009fdb941849728d053c9fb04c5 2024-11-26T11:15:57,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/90115e13f95b4282a4d2d1fd61a9c918 is 50, key is test_row_0/B:col10/1732619757514/Put/seqid=0 2024-11-26T11:15:57,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742101_1277 (size=12301) 2024-11-26T11:15:58,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:58,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:58,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:15:58,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619818183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619818193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:58,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/90115e13f95b4282a4d2d1fd61a9c918 2024-11-26T11:15:58,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/6191d9ef7e7940e3bcd9851632d80807 is 50, key is test_row_0/C:col10/1732619757514/Put/seqid=0 2024-11-26T11:15:58,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742102_1278 (size=12301) 2024-11-26T11:15:58,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:15:58,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:15:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619818693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:58,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619818703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:58,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/6191d9ef7e7940e3bcd9851632d80807 2024-11-26T11:15:58,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/e0cc7009fdb941849728d053c9fb04c5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/e0cc7009fdb941849728d053c9fb04c5 2024-11-26T11:15:58,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/e0cc7009fdb941849728d053c9fb04c5, entries=200, sequenceid=797, filesize=14.4 K 2024-11-26T11:15:58,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/90115e13f95b4282a4d2d1fd61a9c918 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/90115e13f95b4282a4d2d1fd61a9c918 2024-11-26T11:15:58,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/90115e13f95b4282a4d2d1fd61a9c918, entries=150, sequenceid=797, filesize=12.0 K 2024-11-26T11:15:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/6191d9ef7e7940e3bcd9851632d80807 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6191d9ef7e7940e3bcd9851632d80807 2024-11-26T11:15:58,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6191d9ef7e7940e3bcd9851632d80807, entries=150, sequenceid=797, filesize=12.0 K 2024-11-26T11:15:58,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for de67c03f4cecf266dfeecbd54a45d9f0 in 1255ms, sequenceid=797, compaction requested=true 2024-11-26T11:15:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:15:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:58,770 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:58,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:15:58,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:58,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:15:58,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T11:15:58,770 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:58,771 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:58,771 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:15:58,771 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:58,771 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:15:58,771 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,771 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,771 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/47efedaa019a4b83a06c784aede0caeb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/d9a0919e0b1e4cc28da8cd985e9c964c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/e0cc7009fdb941849728d053c9fb04c5] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=39.0 K 2024-11-26T11:15:58,771 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b81064cf436844cebacfcfab0a891461, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9440d144fd3647ddaecf2328e7c237be, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/90115e13f95b4282a4d2d1fd61a9c918] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.6 K 2024-11-26T11:15:58,771 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b81064cf436844cebacfcfab0a891461, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=745, earliestPutTs=1732619754979 2024-11-26T11:15:58,771 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47efedaa019a4b83a06c784aede0caeb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=745, earliestPutTs=1732619754979 2024-11-26T11:15:58,772 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9a0919e0b1e4cc28da8cd985e9c964c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=782, earliestPutTs=1732619755059 2024-11-26T11:15:58,772 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9440d144fd3647ddaecf2328e7c237be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=782, earliestPutTs=1732619755059 2024-11-26T11:15:58,772 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0cc7009fdb941849728d053c9fb04c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=797, earliestPutTs=1732619757175 2024-11-26T11:15:58,772 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 90115e13f95b4282a4d2d1fd61a9c918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=797, earliestPutTs=1732619757175 2024-11-26T11:15:58,785 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:58,785 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:58,785 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/51cf9bb3db1d4dbd908debcb9ab46ad6 is 50, key is test_row_0/A:col10/1732619757514/Put/seqid=0 2024-11-26T11:15:58,785 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/1659bca0850b4b54a603a67d3bc0f280 is 50, key is test_row_0/B:col10/1732619757514/Put/seqid=0 2024-11-26T11:15:58,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742104_1280 (size=12983) 2024-11-26T11:15:58,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742103_1279 (size=12983) 2024-11-26T11:15:58,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:58,796 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:58,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/00e468b35f0c4835bc0d13df175501d1 is 50, key is test_row_0/A:col10/1732619757552/Put/seqid=0 2024-11-26T11:15:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742105_1281 (size=12301) 2024-11-26T11:15:59,253 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=816 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/00e468b35f0c4835bc0d13df175501d1 2024-11-26T11:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:59,255 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/51cf9bb3db1d4dbd908debcb9ab46ad6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/51cf9bb3db1d4dbd908debcb9ab46ad6 2024-11-26T11:15:59,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:15:59,256 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/1659bca0850b4b54a603a67d3bc0f280 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/1659bca0850b4b54a603a67d3bc0f280 2024-11-26T11:15:59,265 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into 51cf9bb3db1d4dbd908debcb9ab46ad6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:59,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:59,265 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619758769; duration=0sec 2024-11-26T11:15:59,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:15:59,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:15:59,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:15:59,267 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:15:59,267 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:15:59,267 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:59,267 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/a2769fb6b9104d9eaf6af7c3f54deea9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/87ffe6a0daff41c1bd3bbaab4970d2b4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6191d9ef7e7940e3bcd9851632d80807] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.6 K 2024-11-26T11:15:59,268 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2769fb6b9104d9eaf6af7c3f54deea9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=745, earliestPutTs=1732619754979 2024-11-26T11:15:59,269 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87ffe6a0daff41c1bd3bbaab4970d2b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=782, earliestPutTs=1732619755059 2024-11-26T11:15:59,269 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into 1659bca0850b4b54a603a67d3bc0f280(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:59,269 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:59,269 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619758769; duration=0sec 2024-11-26T11:15:59,270 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:59,270 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:15:59,270 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6191d9ef7e7940e3bcd9851632d80807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=797, earliestPutTs=1732619757175 2024-11-26T11:15:59,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/fcfbcfa9eaa44934a6f42c8a22b59eaf is 50, key is test_row_0/B:col10/1732619757552/Put/seqid=0 2024-11-26T11:15:59,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742106_1282 (size=12301) 2024-11-26T11:15:59,284 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=816 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/fcfbcfa9eaa44934a6f42c8a22b59eaf 2024-11-26T11:15:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/aa8c5e5d16c94eb6aac2c2318b5795b1 is 50, key is test_row_0/C:col10/1732619757552/Put/seqid=0 2024-11-26T11:15:59,298 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#238 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:15:59,298 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/89dd8a25a3404fe99b47e3adfe1c6862 is 50, key is test_row_0/C:col10/1732619757514/Put/seqid=0 2024-11-26T11:15:59,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742108_1284 (size=12301) 2024-11-26T11:15:59,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619819310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742107_1283 (size=12983) 2024-11-26T11:15:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619819423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:15:59,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619819633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,703 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=816 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/aa8c5e5d16c94eb6aac2c2318b5795b1 2024-11-26T11:15:59,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619819702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,706 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/00e468b35f0c4835bc0d13df175501d1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/00e468b35f0c4835bc0d13df175501d1 2024-11-26T11:15:59,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/00e468b35f0c4835bc0d13df175501d1, entries=150, sequenceid=816, filesize=12.0 K 2024-11-26T11:15:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/fcfbcfa9eaa44934a6f42c8a22b59eaf as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/fcfbcfa9eaa44934a6f42c8a22b59eaf 2024-11-26T11:15:59,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/fcfbcfa9eaa44934a6f42c8a22b59eaf, entries=150, sequenceid=816, filesize=12.0 K 2024-11-26T11:15:59,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619819713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/aa8c5e5d16c94eb6aac2c2318b5795b1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/aa8c5e5d16c94eb6aac2c2318b5795b1 2024-11-26T11:15:59,716 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:15:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/aa8c5e5d16c94eb6aac2c2318b5795b1, entries=150, sequenceid=816, filesize=12.0 K 2024-11-26T11:15:59,720 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for de67c03f4cecf266dfeecbd54a45d9f0 in 924ms, sequenceid=816, compaction requested=false 2024-11-26T11:15:59,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:59,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:15:59,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-26T11:15:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-26T11:15:59,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-26T11:15:59,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3030 sec 2024-11-26T11:15:59,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3090 sec 2024-11-26T11:15:59,729 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/89dd8a25a3404fe99b47e3adfe1c6862 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/89dd8a25a3404fe99b47e3adfe1c6862 2024-11-26T11:15:59,734 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into 89dd8a25a3404fe99b47e3adfe1c6862(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:15:59,734 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:15:59,734 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619758770; duration=0sec 2024-11-26T11:15:59,734 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:15:59,734 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:15:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:15:59,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:15:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:15:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:15:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:15:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:15:59,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b448d693cb6c416b92d71816594dbcd9 is 50, key is test_row_0/A:col10/1732619759303/Put/seqid=0 2024-11-26T11:15:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742109_1285 (size=14741) 2024-11-26T11:15:59,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b448d693cb6c416b92d71816594dbcd9 2024-11-26T11:15:59,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/082bc13e7f7d44b299755ce35bba4199 is 50, key is test_row_0/B:col10/1732619759303/Put/seqid=0 2024-11-26T11:15:59,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742110_1286 (size=12301) 2024-11-26T11:15:59,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619819982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619819988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:15:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619819988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:15:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:15:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619820093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619820103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619820103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619820303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619820313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619820313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/082bc13e7f7d44b299755ce35bba4199 2024-11-26T11:16:00,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/f3ccb336701c433486ae79da07f05415 is 50, key is test_row_0/C:col10/1732619759303/Put/seqid=0 2024-11-26T11:16:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742111_1287 (size=12301) 2024-11-26T11:16:00,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=833 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/f3ccb336701c433486ae79da07f05415 2024-11-26T11:16:00,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b448d693cb6c416b92d71816594dbcd9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b448d693cb6c416b92d71816594dbcd9 2024-11-26T11:16:00,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b448d693cb6c416b92d71816594dbcd9, entries=200, sequenceid=833, filesize=14.4 K 2024-11-26T11:16:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/082bc13e7f7d44b299755ce35bba4199 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/082bc13e7f7d44b299755ce35bba4199 2024-11-26T11:16:00,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/082bc13e7f7d44b299755ce35bba4199, entries=150, sequenceid=833, filesize=12.0 K 2024-11-26T11:16:00,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/f3ccb336701c433486ae79da07f05415 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/f3ccb336701c433486ae79da07f05415 2024-11-26T11:16:00,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/f3ccb336701c433486ae79da07f05415, entries=150, sequenceid=833, filesize=12.0 K 2024-11-26T11:16:00,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for de67c03f4cecf266dfeecbd54a45d9f0 in 545ms, sequenceid=833, compaction requested=true 2024-11-26T11:16:00,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:00,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:00,482 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:00,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:00,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:00,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:00,483 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:00,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:00,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:00,487 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:00,487 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:00,488 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:16:00,488 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:16:00,488 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:00,488 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:00,488 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/1659bca0850b4b54a603a67d3bc0f280, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/fcfbcfa9eaa44934a6f42c8a22b59eaf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/082bc13e7f7d44b299755ce35bba4199] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.7 K 2024-11-26T11:16:00,488 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/51cf9bb3db1d4dbd908debcb9ab46ad6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/00e468b35f0c4835bc0d13df175501d1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b448d693cb6c416b92d71816594dbcd9] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=39.1 K 2024-11-26T11:16:00,490 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1659bca0850b4b54a603a67d3bc0f280, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=797, earliestPutTs=1732619757175 2024-11-26T11:16:00,490 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51cf9bb3db1d4dbd908debcb9ab46ad6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=797, earliestPutTs=1732619757175 2024-11-26T11:16:00,490 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting fcfbcfa9eaa44934a6f42c8a22b59eaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=816, earliestPutTs=1732619757550 2024-11-26T11:16:00,491 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 082bc13e7f7d44b299755ce35bba4199, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1732619759294 2024-11-26T11:16:00,491 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e468b35f0c4835bc0d13df175501d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=816, earliestPutTs=1732619757550 2024-11-26T11:16:00,491 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b448d693cb6c416b92d71816594dbcd9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1732619759294 2024-11-26T11:16:00,514 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:00,514 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:00,514 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/30280b3e8ac94f429621a9aa33a9f5e2 is 50, key is test_row_0/B:col10/1732619759303/Put/seqid=0 2024-11-26T11:16:00,515 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/dbe95690ed814c33b35d877a1d09511f is 50, key is test_row_0/A:col10/1732619759303/Put/seqid=0 2024-11-26T11:16:00,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742113_1289 (size=13085) 2024-11-26T11:16:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742112_1288 (size=13085) 2024-11-26T11:16:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:00,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:16:00,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:00,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/055fccdb8d2f449aa1eea24f9c30ad37 is 50, key is test_row_0/A:col10/1732619759986/Put/seqid=0 2024-11-26T11:16:00,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742114_1290 (size=14741) 2024-11-26T11:16:00,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/055fccdb8d2f449aa1eea24f9c30ad37 2024-11-26T11:16:00,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/ae9c9b1d44f7462f83c06bc874d06f20 is 50, key is test_row_0/B:col10/1732619759986/Put/seqid=0 2024-11-26T11:16:00,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742115_1291 (size=12301) 2024-11-26T11:16:00,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619820672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619820679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619820683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619820783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619820792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619820793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:00,962 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/30280b3e8ac94f429621a9aa33a9f5e2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/30280b3e8ac94f429621a9aa33a9f5e2 2024-11-26T11:16:00,969 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into 30280b3e8ac94f429621a9aa33a9f5e2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:00,969 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:00,969 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619760483; duration=0sec 2024-11-26T11:16:00,969 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:00,970 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:16:00,970 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:00,973 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:00,973 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:16:00,974 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:00,974 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/89dd8a25a3404fe99b47e3adfe1c6862, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/aa8c5e5d16c94eb6aac2c2318b5795b1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/f3ccb336701c433486ae79da07f05415] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.7 K 2024-11-26T11:16:00,974 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 89dd8a25a3404fe99b47e3adfe1c6862, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=797, earliestPutTs=1732619757175 2024-11-26T11:16:00,976 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting aa8c5e5d16c94eb6aac2c2318b5795b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=816, earliestPutTs=1732619757550 2024-11-26T11:16:00,979 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ccb336701c433486ae79da07f05415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1732619759294 2024-11-26T11:16:00,986 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/dbe95690ed814c33b35d877a1d09511f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/dbe95690ed814c33b35d877a1d09511f 2024-11-26T11:16:00,995 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into dbe95690ed814c33b35d877a1d09511f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:00,995 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:00,995 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619760482; duration=0sec 2024-11-26T11:16:00,996 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:00,996 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:16:00,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619820993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619821003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619821003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,014 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:01,014 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d9e24ff7c45b49318de7b40d6cee2d30 is 50, key is test_row_0/C:col10/1732619759303/Put/seqid=0 2024-11-26T11:16:01,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742116_1292 (size=13085) 2024-11-26T11:16:01,039 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d9e24ff7c45b49318de7b40d6cee2d30 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d9e24ff7c45b49318de7b40d6cee2d30 2024-11-26T11:16:01,046 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into d9e24ff7c45b49318de7b40d6cee2d30(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:01,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/ae9c9b1d44f7462f83c06bc874d06f20 2024-11-26T11:16:01,046 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:01,046 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619760483; duration=0sec 2024-11-26T11:16:01,046 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:01,046 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:16:01,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/026b586c03b842c48f353c7e7d47e0d5 is 50, key is test_row_0/C:col10/1732619759986/Put/seqid=0 2024-11-26T11:16:01,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742117_1293 (size=12301) 2024-11-26T11:16:01,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/026b586c03b842c48f353c7e7d47e0d5 2024-11-26T11:16:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/055fccdb8d2f449aa1eea24f9c30ad37 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/055fccdb8d2f449aa1eea24f9c30ad37 2024-11-26T11:16:01,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/055fccdb8d2f449aa1eea24f9c30ad37, entries=200, sequenceid=853, filesize=14.4 K 2024-11-26T11:16:01,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/ae9c9b1d44f7462f83c06bc874d06f20 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ae9c9b1d44f7462f83c06bc874d06f20 2024-11-26T11:16:01,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ae9c9b1d44f7462f83c06bc874d06f20, entries=150, sequenceid=853, filesize=12.0 K 2024-11-26T11:16:01,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/026b586c03b842c48f353c7e7d47e0d5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/026b586c03b842c48f353c7e7d47e0d5 2024-11-26T11:16:01,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/026b586c03b842c48f353c7e7d47e0d5, entries=150, sequenceid=853, filesize=12.0 K 2024-11-26T11:16:01,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for de67c03f4cecf266dfeecbd54a45d9f0 in 506ms, sequenceid=853, compaction requested=false 2024-11-26T11:16:01,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:01,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T11:16:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:01,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:01,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/273c2b196ed44ea798c1c1fb741ca59f is 50, key is test_row_0/A:col10/1732619761305/Put/seqid=0 2024-11-26T11:16:01,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742118_1294 (size=14741) 2024-11-26T11:16:01,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619821364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619821364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619821366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619821473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619821483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619821483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T11:16:01,574 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:01,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-26T11:16:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T11:16:01,578 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:01,584 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:01,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T11:16:01,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619821683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619821694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619821694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48194 deadline: 1732619821713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,719 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48262 deadline: 1732619821732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,735 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in de67c03f4cecf266dfeecbd54a45d9f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:01,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T11:16:01,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:01,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:01,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:01,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:01,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:01,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=869 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/273c2b196ed44ea798c1c1fb741ca59f 2024-11-26T11:16:01,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/e5f03d43ee6e4c0587b5e715eadbec65 is 50, key is test_row_0/B:col10/1732619761305/Put/seqid=0 2024-11-26T11:16:01,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742119_1295 (size=12301) 2024-11-26T11:16:01,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=869 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/e5f03d43ee6e4c0587b5e715eadbec65 2024-11-26T11:16:01,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/7927b5d907904719862bfd4b8c60d659 is 50, key is test_row_0/C:col10/1732619761305/Put/seqid=0 2024-11-26T11:16:01,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742120_1296 (size=12301) 2024-11-26T11:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T11:16:01,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T11:16:01,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:01,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:01,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:01,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:01,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:01,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619821993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619822002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619822002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T11:16:02,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:02,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T11:16:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T11:16:02,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=869 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/7927b5d907904719862bfd4b8c60d659 2024-11-26T11:16:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/273c2b196ed44ea798c1c1fb741ca59f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/273c2b196ed44ea798c1c1fb741ca59f 2024-11-26T11:16:02,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/273c2b196ed44ea798c1c1fb741ca59f, entries=200, sequenceid=869, filesize=14.4 K 2024-11-26T11:16:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/e5f03d43ee6e4c0587b5e715eadbec65 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/e5f03d43ee6e4c0587b5e715eadbec65 2024-11-26T11:16:02,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/e5f03d43ee6e4c0587b5e715eadbec65, entries=150, sequenceid=869, filesize=12.0 K 2024-11-26T11:16:02,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/7927b5d907904719862bfd4b8c60d659 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7927b5d907904719862bfd4b8c60d659 2024-11-26T11:16:02,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7927b5d907904719862bfd4b8c60d659, entries=150, sequenceid=869, filesize=12.0 K 2024-11-26T11:16:02,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for de67c03f4cecf266dfeecbd54a45d9f0 in 969ms, sequenceid=869, compaction requested=true 2024-11-26T11:16:02,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:02,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:02,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:02,276 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:02,276 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:02,277 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:02,277 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:02,278 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:16:02,278 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,278 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/dbe95690ed814c33b35d877a1d09511f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/055fccdb8d2f449aa1eea24f9c30ad37, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/273c2b196ed44ea798c1c1fb741ca59f] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=41.6 K 2024-11-26T11:16:02,278 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:16:02,278 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,278 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/30280b3e8ac94f429621a9aa33a9f5e2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ae9c9b1d44f7462f83c06bc874d06f20, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/e5f03d43ee6e4c0587b5e715eadbec65] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.8 K 2024-11-26T11:16:02,278 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 30280b3e8ac94f429621a9aa33a9f5e2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1732619759294 2024-11-26T11:16:02,279 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe95690ed814c33b35d877a1d09511f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1732619759294 2024-11-26T11:16:02,279 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ae9c9b1d44f7462f83c06bc874d06f20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732619759969 2024-11-26T11:16:02,279 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055fccdb8d2f449aa1eea24f9c30ad37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732619759969 2024-11-26T11:16:02,279 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e5f03d43ee6e4c0587b5e715eadbec65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1732619760645 2024-11-26T11:16:02,279 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 273c2b196ed44ea798c1c1fb741ca59f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1732619760629 2024-11-26T11:16:02,290 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:02,290 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/05f1a60369774b9e868e835de6f414bb is 50, key is test_row_0/B:col10/1732619761305/Put/seqid=0 2024-11-26T11:16:02,297 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:02,298 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/f29d328a768d4b4396109c8e57317e1d is 50, key is test_row_0/A:col10/1732619761305/Put/seqid=0 2024-11-26T11:16:02,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742121_1297 (size=13187) 2024-11-26T11:16:02,315 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/05f1a60369774b9e868e835de6f414bb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/05f1a60369774b9e868e835de6f414bb 2024-11-26T11:16:02,320 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into 05f1a60369774b9e868e835de6f414bb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:02,320 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:02,320 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619762276; duration=0sec 2024-11-26T11:16:02,320 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:02,320 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:16:02,320 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:02,322 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:02,322 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:16:02,322 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,323 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d9e24ff7c45b49318de7b40d6cee2d30, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/026b586c03b842c48f353c7e7d47e0d5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7927b5d907904719862bfd4b8c60d659] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.8 K 2024-11-26T11:16:02,324 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e24ff7c45b49318de7b40d6cee2d30, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=833, earliestPutTs=1732619759294 2024-11-26T11:16:02,324 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 026b586c03b842c48f353c7e7d47e0d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=853, earliestPutTs=1732619759969 2024-11-26T11:16:02,324 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7927b5d907904719862bfd4b8c60d659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1732619760645 2024-11-26T11:16:02,332 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:02,333 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/4dafaedd07a44b88b4294d5aff9723b0 is 50, key is test_row_0/C:col10/1732619761305/Put/seqid=0 2024-11-26T11:16:02,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742122_1298 (size=13187) 2024-11-26T11:16:02,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T11:16:02,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:02,356 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:16:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:02,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742123_1299 (size=13187) 2024-11-26T11:16:02,372 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/4dafaedd07a44b88b4294d5aff9723b0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4dafaedd07a44b88b4294d5aff9723b0 2024-11-26T11:16:02,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/2639bae096e5443dbc2830373f245f8a is 50, key is test_row_0/A:col10/1732619761364/Put/seqid=0 2024-11-26T11:16:02,379 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into 4dafaedd07a44b88b4294d5aff9723b0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:02,379 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:02,379 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619762277; duration=0sec 2024-11-26T11:16:02,380 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:02,380 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:16:02,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742124_1300 (size=12301) 2024-11-26T11:16:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:02,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:02,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619822525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619822523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619822526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619822632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619822632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619822632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T11:16:02,737 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/f29d328a768d4b4396109c8e57317e1d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/f29d328a768d4b4396109c8e57317e1d 2024-11-26T11:16:02,741 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into f29d328a768d4b4396109c8e57317e1d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:02,741 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:02,741 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619762276; duration=0sec 2024-11-26T11:16:02,741 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:02,741 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:16:02,786 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=889 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/2639bae096e5443dbc2830373f245f8a 2024-11-26T11:16:02,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/7da1d26ab1f74d098a04b5a3f96ba7fd is 50, key is test_row_0/B:col10/1732619761364/Put/seqid=0 2024-11-26T11:16:02,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742125_1301 (size=12301) 2024-11-26T11:16:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619822842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619822843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619822843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619823153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619823153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619823153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,196 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=889 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/7da1d26ab1f74d098a04b5a3f96ba7fd 2024-11-26T11:16:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/ca117a61aab04905a7e1eff86b6841c4 is 50, key is test_row_0/C:col10/1732619761364/Put/seqid=0 2024-11-26T11:16:03,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742126_1302 (size=12301) 2024-11-26T11:16:03,209 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=889 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/ca117a61aab04905a7e1eff86b6841c4 2024-11-26T11:16:03,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/2639bae096e5443dbc2830373f245f8a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/2639bae096e5443dbc2830373f245f8a 2024-11-26T11:16:03,219 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/2639bae096e5443dbc2830373f245f8a, entries=150, sequenceid=889, filesize=12.0 K 2024-11-26T11:16:03,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/7da1d26ab1f74d098a04b5a3f96ba7fd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/7da1d26ab1f74d098a04b5a3f96ba7fd 2024-11-26T11:16:03,223 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/7da1d26ab1f74d098a04b5a3f96ba7fd, entries=150, sequenceid=889, filesize=12.0 K 2024-11-26T11:16:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/ca117a61aab04905a7e1eff86b6841c4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ca117a61aab04905a7e1eff86b6841c4 2024-11-26T11:16:03,227 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ca117a61aab04905a7e1eff86b6841c4, entries=150, sequenceid=889, filesize=12.0 K 2024-11-26T11:16:03,227 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for de67c03f4cecf266dfeecbd54a45d9f0 in 872ms, sequenceid=889, compaction requested=false 2024-11-26T11:16:03,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:03,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:03,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-26T11:16:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-26T11:16:03,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T11:16:03,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-26T11:16:03,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6570 sec 2024-11-26T11:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:03,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:03,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b14c932002f845cd90c8d5100f19d34b is 50, key is test_row_0/A:col10/1732619762525/Put/seqid=0 2024-11-26T11:16:03,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742127_1303 (size=14741) 2024-11-26T11:16:03,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619823685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619823690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619823691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T11:16:03,723 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:03,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-26T11:16:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T11:16:03,725 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:03,725 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:03,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:03,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619823793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619823802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619823802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T11:16:03,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T11:16:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:03,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:03,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:03,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:03,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:04,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619824003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619824012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619824012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T11:16:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:04,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T11:16:04,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=906 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b14c932002f845cd90c8d5100f19d34b 2024-11-26T11:16:04,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/9be550b6f9574cbb816504c41e9dd407 is 50, key is test_row_0/B:col10/1732619762525/Put/seqid=0 2024-11-26T11:16:04,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742128_1304 (size=12301) 2024-11-26T11:16:04,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=906 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/9be550b6f9574cbb816504c41e9dd407 2024-11-26T11:16:04,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/4beea6105c3f420c8f5e304ce29649f0 is 50, key is test_row_0/C:col10/1732619762525/Put/seqid=0 2024-11-26T11:16:04,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742129_1305 (size=12301) 2024-11-26T11:16:04,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=906 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/4beea6105c3f420c8f5e304ce29649f0 2024-11-26T11:16:04,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/b14c932002f845cd90c8d5100f19d34b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b14c932002f845cd90c8d5100f19d34b 2024-11-26T11:16:04,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b14c932002f845cd90c8d5100f19d34b, entries=200, sequenceid=906, filesize=14.4 K 2024-11-26T11:16:04,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/9be550b6f9574cbb816504c41e9dd407 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9be550b6f9574cbb816504c41e9dd407 2024-11-26T11:16:04,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9be550b6f9574cbb816504c41e9dd407, entries=150, sequenceid=906, filesize=12.0 K 2024-11-26T11:16:04,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/4beea6105c3f420c8f5e304ce29649f0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4beea6105c3f420c8f5e304ce29649f0 2024-11-26T11:16:04,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4beea6105c3f420c8f5e304ce29649f0, entries=150, sequenceid=906, filesize=12.0 K 2024-11-26T11:16:04,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for de67c03f4cecf266dfeecbd54a45d9f0 in 444ms, sequenceid=906, compaction requested=true 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:04,110 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:04,110 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:04,111 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:04,111 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/A is initiating minor compaction (all files) 2024-11-26T11:16:04,111 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/A in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:04,111 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/f29d328a768d4b4396109c8e57317e1d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/2639bae096e5443dbc2830373f245f8a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b14c932002f845cd90c8d5100f19d34b] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=39.3 K 2024-11-26T11:16:04,112 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:04,112 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/B is initiating minor compaction (all files) 2024-11-26T11:16:04,112 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/B in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:04,112 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/05f1a60369774b9e868e835de6f414bb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/7da1d26ab1f74d098a04b5a3f96ba7fd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9be550b6f9574cbb816504c41e9dd407] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.9 K 2024-11-26T11:16:04,112 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f1a60369774b9e868e835de6f414bb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1732619760645 2024-11-26T11:16:04,112 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f29d328a768d4b4396109c8e57317e1d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1732619760645 2024-11-26T11:16:04,113 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da1d26ab1f74d098a04b5a3f96ba7fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732619761345 2024-11-26T11:16:04,113 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2639bae096e5443dbc2830373f245f8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732619761345 2024-11-26T11:16:04,113 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be550b6f9574cbb816504c41e9dd407, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=906, earliestPutTs=1732619762525 2024-11-26T11:16:04,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14c932002f845cd90c8d5100f19d34b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=906, earliestPutTs=1732619762522 2024-11-26T11:16:04,129 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:04,129 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/0963aacfa06b428b84df81fb90df89c7 is 50, key is test_row_0/B:col10/1732619762525/Put/seqid=0 2024-11-26T11:16:04,132 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:04,132 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/eb7cb753a0d443b9afa7b4ce41c02c99 is 50, key is test_row_0/A:col10/1732619762525/Put/seqid=0 2024-11-26T11:16:04,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742130_1306 (size=13289) 2024-11-26T11:16:04,143 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/0963aacfa06b428b84df81fb90df89c7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0963aacfa06b428b84df81fb90df89c7 2024-11-26T11:16:04,148 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/B of de67c03f4cecf266dfeecbd54a45d9f0 into 0963aacfa06b428b84df81fb90df89c7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:04,148 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:04,148 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/B, priority=13, startTime=1732619764110; duration=0sec 2024-11-26T11:16:04,148 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:04,149 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:16:04,149 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:04,150 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:04,150 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): de67c03f4cecf266dfeecbd54a45d9f0/C is initiating minor compaction (all files) 2024-11-26T11:16:04,150 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de67c03f4cecf266dfeecbd54a45d9f0/C in TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:04,150 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4dafaedd07a44b88b4294d5aff9723b0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ca117a61aab04905a7e1eff86b6841c4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4beea6105c3f420c8f5e304ce29649f0] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp, totalSize=36.9 K 2024-11-26T11:16:04,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742131_1307 (size=13289) 2024-11-26T11:16:04,151 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dafaedd07a44b88b4294d5aff9723b0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=869, earliestPutTs=1732619760645 2024-11-26T11:16:04,151 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ca117a61aab04905a7e1eff86b6841c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=889, earliestPutTs=1732619761345 2024-11-26T11:16:04,151 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4beea6105c3f420c8f5e304ce29649f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=906, earliestPutTs=1732619762525 2024-11-26T11:16:04,157 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de67c03f4cecf266dfeecbd54a45d9f0#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:04,160 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/58b2acb020144ecd9d091de67a65d1c9 is 50, key is test_row_0/C:col10/1732619762525/Put/seqid=0 2024-11-26T11:16:04,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742132_1308 (size=13289) 2024-11-26T11:16:04,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:04,183 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:04,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:04,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/727b31045c664f8cbd8204ae7003d467 is 50, key is test_row_0/A:col10/1732619763684/Put/seqid=0 2024-11-26T11:16:04,263 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T11:16:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742133_1309 (size=12301) 2024-11-26T11:16:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:04,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. as already flushing 2024-11-26T11:16:04,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619824339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619824340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619824341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T11:16:04,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619824453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619824453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619824453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,554 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/eb7cb753a0d443b9afa7b4ce41c02c99 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eb7cb753a0d443b9afa7b4ce41c02c99 2024-11-26T11:16:04,559 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/A of de67c03f4cecf266dfeecbd54a45d9f0 into eb7cb753a0d443b9afa7b4ce41c02c99(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:04,559 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:04,559 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/A, priority=13, startTime=1732619764110; duration=0sec 2024-11-26T11:16:04,559 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:04,559 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:16:04,578 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/58b2acb020144ecd9d091de67a65d1c9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/58b2acb020144ecd9d091de67a65d1c9 2024-11-26T11:16:04,583 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de67c03f4cecf266dfeecbd54a45d9f0/C of de67c03f4cecf266dfeecbd54a45d9f0 into 58b2acb020144ecd9d091de67a65d1c9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:04,583 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:04,583 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., storeName=de67c03f4cecf266dfeecbd54a45d9f0/C, priority=13, startTime=1732619764110; duration=0sec 2024-11-26T11:16:04,583 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:04,583 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:16:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619824663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619824663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619824663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=925 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/727b31045c664f8cbd8204ae7003d467 2024-11-26T11:16:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/79c7cea568184220b00ccd6d78fde435 is 50, key is test_row_0/B:col10/1732619763684/Put/seqid=0 2024-11-26T11:16:04,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742134_1310 (size=12301) 2024-11-26T11:16:04,735 INFO [Thread-1049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1049. 2024-11-26T11:16:04,735 DEBUG [Thread-1049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:04,735 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,735 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:04,736 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-26T11:16:04,736 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:04,736 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,736 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:04,737 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-26T11:16:04,737 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:04,737 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,737 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:04,739 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-26T11:16:04,739 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:04,739 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,739 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:04,740 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-26T11:16:04,740 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:04,740 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,740 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:04,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T11:16:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619824973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619824973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619824973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:05,132 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=925 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/79c7cea568184220b00ccd6d78fde435 2024-11-26T11:16:05,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d6db3f47f53d499d9fc15537d00e75b3 is 50, key is test_row_0/C:col10/1732619763684/Put/seqid=0 2024-11-26T11:16:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742135_1311 (size=12301) 2024-11-26T11:16:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48224 deadline: 1732619825483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48248 deadline: 1732619825483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48232 deadline: 1732619825483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:05,548 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=925 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d6db3f47f53d499d9fc15537d00e75b3 2024-11-26T11:16:05,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/727b31045c664f8cbd8204ae7003d467 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/727b31045c664f8cbd8204ae7003d467 2024-11-26T11:16:05,560 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/727b31045c664f8cbd8204ae7003d467, entries=150, sequenceid=925, filesize=12.0 K 2024-11-26T11:16:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/79c7cea568184220b00ccd6d78fde435 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/79c7cea568184220b00ccd6d78fde435 2024-11-26T11:16:05,567 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/79c7cea568184220b00ccd6d78fde435, entries=150, sequenceid=925, filesize=12.0 K 2024-11-26T11:16:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/d6db3f47f53d499d9fc15537d00e75b3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d6db3f47f53d499d9fc15537d00e75b3 2024-11-26T11:16:05,572 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d6db3f47f53d499d9fc15537d00e75b3, entries=150, sequenceid=925, filesize=12.0 K 2024-11-26T11:16:05,573 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for de67c03f4cecf266dfeecbd54a45d9f0 in 1390ms, sequenceid=925, compaction requested=false 2024-11-26T11:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:05,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-26T11:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-26T11:16:05,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T11:16:05,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-26T11:16:05,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8530 sec 2024-11-26T11:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:05,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T11:16:05,756 INFO [Thread-1034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1034. 2024-11-26T11:16:05,756 DEBUG [Thread-1034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:05,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:05,756 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:05,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:05,757 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:05,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:05,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:05,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3bb46b232f0f4effa5a01be596535fea is 50, key is test_row_0/A:col10/1732619764338/Put/seqid=0 2024-11-26T11:16:05,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742136_1312 (size=12301) 2024-11-26T11:16:05,775 INFO [Thread-1036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1036. 2024-11-26T11:16:05,775 DEBUG [Thread-1036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:05,775 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:05,775 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:05,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T11:16:05,873 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:06,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3bb46b232f0f4effa5a01be596535fea 2024-11-26T11:16:06,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d0b9d079b6ed4b4abffbeb12d0e4791a is 50, key is test_row_0/B:col10/1732619764338/Put/seqid=0 2024-11-26T11:16:06,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742137_1313 (size=12301) 2024-11-26T11:16:06,494 INFO [Thread-1032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1032. 2024-11-26T11:16:06,495 DEBUG [Thread-1032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:06,495 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,495 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:06,504 INFO [Thread-1038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1038. 2024-11-26T11:16:06,504 INFO [Thread-1030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1030. 2024-11-26T11:16:06,504 DEBUG [Thread-1030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:06,504 DEBUG [Thread-1038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:06,504 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,504 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,505 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,505 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:06,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 201 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 188 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 176 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2962 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2985 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2979 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2927 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2954 2024-11-26T11:16:06,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T11:16:06,505 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:16:06,505 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:16:06,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:06,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:06,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T11:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T11:16:06,508 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619766508"}]},"ts":"1732619766508"} 2024-11-26T11:16:06,509 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T11:16:06,509 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T11:16:06,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:16:06,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, UNASSIGN}] 2024-11-26T11:16:06,512 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, UNASSIGN 2024-11-26T11:16:06,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=de67c03f4cecf266dfeecbd54a45d9f0, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:06,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, UNASSIGN because future has completed 2024-11-26T11:16:06,514 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:16:06,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:16:06,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d0b9d079b6ed4b4abffbeb12d0e4791a 2024-11-26T11:16:06,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/9af9985091764b9594c27cb9eff73c93 is 50, key is test_row_0/C:col10/1732619764338/Put/seqid=0 2024-11-26T11:16:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742138_1314 (size=12301) 2024-11-26T11:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T11:16:06,667 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:06,667 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:16:06,668 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing de67c03f4cecf266dfeecbd54a45d9f0, disabling compactions & flushes 2024-11-26T11:16:06,668 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T11:16:06,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/9af9985091764b9594c27cb9eff73c93 2024-11-26T11:16:06,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/3bb46b232f0f4effa5a01be596535fea as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3bb46b232f0f4effa5a01be596535fea 2024-11-26T11:16:06,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3bb46b232f0f4effa5a01be596535fea, entries=150, sequenceid=942, filesize=12.0 K 2024-11-26T11:16:06,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/d0b9d079b6ed4b4abffbeb12d0e4791a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0b9d079b6ed4b4abffbeb12d0e4791a 2024-11-26T11:16:06,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0b9d079b6ed4b4abffbeb12d0e4791a, entries=150, sequenceid=942, filesize=12.0 K 2024-11-26T11:16:07,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/9af9985091764b9594c27cb9eff73c93 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/9af9985091764b9594c27cb9eff73c93 2024-11-26T11:16:07,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/9af9985091764b9594c27cb9eff73c93, entries=150, sequenceid=942, filesize=12.0 K 2024-11-26T11:16:07,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for de67c03f4cecf266dfeecbd54a45d9f0 in 1248ms, sequenceid=942, compaction requested=true 2024-11-26T11:16:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de67c03f4cecf266dfeecbd54a45d9f0: 2024-11-26T11:16:07,004 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:07,004 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:07,004 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. after waiting 0 ms 2024-11-26T11:16:07,004 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:07,004 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. because compaction request was cancelled 2024-11-26T11:16:07,004 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:A 2024-11-26T11:16:07,004 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing de67c03f4cecf266dfeecbd54a45d9f0 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T11:16:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:A, priority=-2147483648, current under compaction store size is 0 2024-11-26T11:16:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:B, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:07,004 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. because compaction request was cancelled 2024-11-26T11:16:07,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:07,004 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:B 2024-11-26T11:16:07,005 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. because compaction request was cancelled 2024-11-26T11:16:07,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de67c03f4cecf266dfeecbd54a45d9f0:C, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:07,005 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de67c03f4cecf266dfeecbd54a45d9f0:C 2024-11-26T11:16:07,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:07,005 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=A 2024-11-26T11:16:07,005 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:07,005 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=B 2024-11-26T11:16:07,005 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:07,005 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de67c03f4cecf266dfeecbd54a45d9f0, store=C 2024-11-26T11:16:07,005 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:07,008 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/23b986ba56a145439ce86ecc6f6c869f is 50, key is test_row_1/A:col10/1732619766503/Put/seqid=0 2024-11-26T11:16:07,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742139_1315 (size=9857) 2024-11-26T11:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T11:16:07,412 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=949 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/23b986ba56a145439ce86ecc6f6c869f 2024-11-26T11:16:07,418 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/3bf715127b72468b8091fae8e8ac0162 is 50, key is test_row_1/B:col10/1732619766503/Put/seqid=0 2024-11-26T11:16:07,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742140_1316 (size=9857) 2024-11-26T11:16:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T11:16:07,825 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=949 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/3bf715127b72468b8091fae8e8ac0162 2024-11-26T11:16:07,834 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/25dc414e804c4a19937e1f49c23e19e6 is 50, key is test_row_1/C:col10/1732619766503/Put/seqid=0 2024-11-26T11:16:07,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742141_1317 (size=9857) 2024-11-26T11:16:08,239 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=949 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/25dc414e804c4a19937e1f49c23e19e6 2024-11-26T11:16:08,243 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/A/23b986ba56a145439ce86ecc6f6c869f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/23b986ba56a145439ce86ecc6f6c869f 2024-11-26T11:16:08,248 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/23b986ba56a145439ce86ecc6f6c869f, entries=100, sequenceid=949, filesize=9.6 K 2024-11-26T11:16:08,250 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/B/3bf715127b72468b8091fae8e8ac0162 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/3bf715127b72468b8091fae8e8ac0162 2024-11-26T11:16:08,255 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/3bf715127b72468b8091fae8e8ac0162, entries=100, sequenceid=949, filesize=9.6 K 2024-11-26T11:16:08,256 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/.tmp/C/25dc414e804c4a19937e1f49c23e19e6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/25dc414e804c4a19937e1f49c23e19e6 2024-11-26T11:16:08,261 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/25dc414e804c4a19937e1f49c23e19e6, entries=100, sequenceid=949, filesize=9.6 K 2024-11-26T11:16:08,262 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for de67c03f4cecf266dfeecbd54a45d9f0 in 1258ms, sequenceid=949, compaction requested=true 2024-11-26T11:16:08,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/6c4a18364e464f029a46dffbe9f620c6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/162c36bb5289458aa2a9945acd58466b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/9c4c8f0291c343d2991fcb606de58fbd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/739fa900a7054c0681d2c0f229bed1f3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eef6152e325d4855abc111984eca994d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b8de98287d14df8804992349b4f278c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/09d653af312e419abc34bc4e742158d3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/538dc808fa48437c86910d600e67a8dc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/adbf7ab0f9ff421a865d8a5e195a23b7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b326ba3e83804affbba186c7873a5383, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/537d8dff68c541c68ac572584db05e5e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b1cb81fa094473d953fa85eacadad71, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b889911747f34b37a6db372b1754a89b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/721f0f069ac4497f8397ab31b5d28f2a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/0b1c43541a754f46b4d3c71c40633460, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/8509b84a7f214449b1643c3c686acb77, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/ee22068760be4254b03fef9354ac6f85, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/47efedaa019a4b83a06c784aede0caeb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/d9a0919e0b1e4cc28da8cd985e9c964c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/e0cc7009fdb941849728d053c9fb04c5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/51cf9bb3db1d4dbd908debcb9ab46ad6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/00e468b35f0c4835bc0d13df175501d1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b448d693cb6c416b92d71816594dbcd9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/dbe95690ed814c33b35d877a1d09511f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/055fccdb8d2f449aa1eea24f9c30ad37, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/273c2b196ed44ea798c1c1fb741ca59f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/f29d328a768d4b4396109c8e57317e1d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/2639bae096e5443dbc2830373f245f8a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b14c932002f845cd90c8d5100f19d34b] to archive 2024-11-26T11:16:08,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:16:08,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/6c4a18364e464f029a46dffbe9f620c6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/6c4a18364e464f029a46dffbe9f620c6 2024-11-26T11:16:08,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/162c36bb5289458aa2a9945acd58466b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/162c36bb5289458aa2a9945acd58466b 2024-11-26T11:16:08,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/9c4c8f0291c343d2991fcb606de58fbd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/9c4c8f0291c343d2991fcb606de58fbd 2024-11-26T11:16:08,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/739fa900a7054c0681d2c0f229bed1f3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/739fa900a7054c0681d2c0f229bed1f3 2024-11-26T11:16:08,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eef6152e325d4855abc111984eca994d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eef6152e325d4855abc111984eca994d 2024-11-26T11:16:08,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b8de98287d14df8804992349b4f278c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b8de98287d14df8804992349b4f278c 2024-11-26T11:16:08,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/09d653af312e419abc34bc4e742158d3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/09d653af312e419abc34bc4e742158d3 2024-11-26T11:16:08,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/538dc808fa48437c86910d600e67a8dc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/538dc808fa48437c86910d600e67a8dc 2024-11-26T11:16:08,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/adbf7ab0f9ff421a865d8a5e195a23b7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/adbf7ab0f9ff421a865d8a5e195a23b7 2024-11-26T11:16:08,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b326ba3e83804affbba186c7873a5383 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b326ba3e83804affbba186c7873a5383 2024-11-26T11:16:08,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/537d8dff68c541c68ac572584db05e5e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/537d8dff68c541c68ac572584db05e5e 2024-11-26T11:16:08,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b1cb81fa094473d953fa85eacadad71 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3b1cb81fa094473d953fa85eacadad71 2024-11-26T11:16:08,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b889911747f34b37a6db372b1754a89b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b889911747f34b37a6db372b1754a89b 2024-11-26T11:16:08,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/721f0f069ac4497f8397ab31b5d28f2a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/721f0f069ac4497f8397ab31b5d28f2a 2024-11-26T11:16:08,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/0b1c43541a754f46b4d3c71c40633460 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/0b1c43541a754f46b4d3c71c40633460 2024-11-26T11:16:08,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/8509b84a7f214449b1643c3c686acb77 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/8509b84a7f214449b1643c3c686acb77 2024-11-26T11:16:08,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/ee22068760be4254b03fef9354ac6f85 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/ee22068760be4254b03fef9354ac6f85 2024-11-26T11:16:08,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/47efedaa019a4b83a06c784aede0caeb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/47efedaa019a4b83a06c784aede0caeb 2024-11-26T11:16:08,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/d9a0919e0b1e4cc28da8cd985e9c964c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/d9a0919e0b1e4cc28da8cd985e9c964c 2024-11-26T11:16:08,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/e0cc7009fdb941849728d053c9fb04c5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/e0cc7009fdb941849728d053c9fb04c5 2024-11-26T11:16:08,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/51cf9bb3db1d4dbd908debcb9ab46ad6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/51cf9bb3db1d4dbd908debcb9ab46ad6 2024-11-26T11:16:08,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/00e468b35f0c4835bc0d13df175501d1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/00e468b35f0c4835bc0d13df175501d1 2024-11-26T11:16:08,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b448d693cb6c416b92d71816594dbcd9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b448d693cb6c416b92d71816594dbcd9 2024-11-26T11:16:08,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/dbe95690ed814c33b35d877a1d09511f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/dbe95690ed814c33b35d877a1d09511f 2024-11-26T11:16:08,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/055fccdb8d2f449aa1eea24f9c30ad37 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/055fccdb8d2f449aa1eea24f9c30ad37 2024-11-26T11:16:08,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/273c2b196ed44ea798c1c1fb741ca59f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/273c2b196ed44ea798c1c1fb741ca59f 2024-11-26T11:16:08,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/f29d328a768d4b4396109c8e57317e1d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/f29d328a768d4b4396109c8e57317e1d 2024-11-26T11:16:08,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/2639bae096e5443dbc2830373f245f8a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/2639bae096e5443dbc2830373f245f8a 2024-11-26T11:16:08,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b14c932002f845cd90c8d5100f19d34b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/b14c932002f845cd90c8d5100f19d34b 2024-11-26T11:16:08,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/dd283a0d38814e538a22c42a4cb0d4f0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/2c90743107964b068576c6b13e7a2add, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/726f3857e6624ef39403f9fa3f0676c5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0db090d4ef99420787bd6768394a364d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/4d8754ec38094dbfa1dcdcd3a69b9f68, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/843d3ace560f43f1afbe2cf1e0f878d9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/47dea3d986b94b36b4f27d0bac6edb6e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/6067188b75d345d581c9f1e4731de466, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/8321a6a2888344e49468dd583a9e8d7a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d65f904839f44e42888334a7acb0060b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/f293ae4c0cff4bcd99d3c10456dc41ec, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/87606aef66624fe4b6d94c859138edc6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/befd55f6efdf485db31d3819012c9bf8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ed508724fb8d40fc8400d721e403cd7a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0a9f63cd7eb472cb170049ec18332bf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/a37b6aa67bae45cc9f8d722749a4d4f3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b81064cf436844cebacfcfab0a891461, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b600e440c6bc4f8c9c43b2acdc85a0a0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9440d144fd3647ddaecf2328e7c237be, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/1659bca0850b4b54a603a67d3bc0f280, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/90115e13f95b4282a4d2d1fd61a9c918, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/fcfbcfa9eaa44934a6f42c8a22b59eaf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/30280b3e8ac94f429621a9aa33a9f5e2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/082bc13e7f7d44b299755ce35bba4199, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ae9c9b1d44f7462f83c06bc874d06f20, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/05f1a60369774b9e868e835de6f414bb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/e5f03d43ee6e4c0587b5e715eadbec65, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/7da1d26ab1f74d098a04b5a3f96ba7fd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9be550b6f9574cbb816504c41e9dd407] to archive 2024-11-26T11:16:08,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:16:08,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/dd283a0d38814e538a22c42a4cb0d4f0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/dd283a0d38814e538a22c42a4cb0d4f0 2024-11-26T11:16:08,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/2c90743107964b068576c6b13e7a2add to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/2c90743107964b068576c6b13e7a2add 2024-11-26T11:16:08,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/726f3857e6624ef39403f9fa3f0676c5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/726f3857e6624ef39403f9fa3f0676c5 2024-11-26T11:16:08,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0db090d4ef99420787bd6768394a364d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0db090d4ef99420787bd6768394a364d 2024-11-26T11:16:08,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/4d8754ec38094dbfa1dcdcd3a69b9f68 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/4d8754ec38094dbfa1dcdcd3a69b9f68 2024-11-26T11:16:08,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/843d3ace560f43f1afbe2cf1e0f878d9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/843d3ace560f43f1afbe2cf1e0f878d9 2024-11-26T11:16:08,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/47dea3d986b94b36b4f27d0bac6edb6e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/47dea3d986b94b36b4f27d0bac6edb6e 2024-11-26T11:16:08,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/6067188b75d345d581c9f1e4731de466 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/6067188b75d345d581c9f1e4731de466 2024-11-26T11:16:08,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/8321a6a2888344e49468dd583a9e8d7a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/8321a6a2888344e49468dd583a9e8d7a 2024-11-26T11:16:08,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d65f904839f44e42888334a7acb0060b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d65f904839f44e42888334a7acb0060b 2024-11-26T11:16:08,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/f293ae4c0cff4bcd99d3c10456dc41ec to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/f293ae4c0cff4bcd99d3c10456dc41ec 2024-11-26T11:16:08,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/87606aef66624fe4b6d94c859138edc6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/87606aef66624fe4b6d94c859138edc6 2024-11-26T11:16:08,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/befd55f6efdf485db31d3819012c9bf8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/befd55f6efdf485db31d3819012c9bf8 2024-11-26T11:16:08,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ed508724fb8d40fc8400d721e403cd7a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ed508724fb8d40fc8400d721e403cd7a 2024-11-26T11:16:08,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0a9f63cd7eb472cb170049ec18332bf to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0a9f63cd7eb472cb170049ec18332bf 2024-11-26T11:16:08,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/a37b6aa67bae45cc9f8d722749a4d4f3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/a37b6aa67bae45cc9f8d722749a4d4f3 2024-11-26T11:16:08,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b81064cf436844cebacfcfab0a891461 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b81064cf436844cebacfcfab0a891461 2024-11-26T11:16:08,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b600e440c6bc4f8c9c43b2acdc85a0a0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/b600e440c6bc4f8c9c43b2acdc85a0a0 2024-11-26T11:16:08,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9440d144fd3647ddaecf2328e7c237be to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9440d144fd3647ddaecf2328e7c237be 2024-11-26T11:16:08,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/1659bca0850b4b54a603a67d3bc0f280 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/1659bca0850b4b54a603a67d3bc0f280 2024-11-26T11:16:08,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/90115e13f95b4282a4d2d1fd61a9c918 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/90115e13f95b4282a4d2d1fd61a9c918 2024-11-26T11:16:08,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/fcfbcfa9eaa44934a6f42c8a22b59eaf to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/fcfbcfa9eaa44934a6f42c8a22b59eaf 2024-11-26T11:16:08,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/30280b3e8ac94f429621a9aa33a9f5e2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/30280b3e8ac94f429621a9aa33a9f5e2 2024-11-26T11:16:08,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/082bc13e7f7d44b299755ce35bba4199 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/082bc13e7f7d44b299755ce35bba4199 2024-11-26T11:16:08,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ae9c9b1d44f7462f83c06bc874d06f20 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/ae9c9b1d44f7462f83c06bc874d06f20 2024-11-26T11:16:08,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/05f1a60369774b9e868e835de6f414bb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/05f1a60369774b9e868e835de6f414bb 2024-11-26T11:16:08,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/e5f03d43ee6e4c0587b5e715eadbec65 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/e5f03d43ee6e4c0587b5e715eadbec65 2024-11-26T11:16:08,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/7da1d26ab1f74d098a04b5a3f96ba7fd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/7da1d26ab1f74d098a04b5a3f96ba7fd 2024-11-26T11:16:08,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9be550b6f9574cbb816504c41e9dd407 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/9be550b6f9574cbb816504c41e9dd407 2024-11-26T11:16:08,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/e28f51417c9545baaa5a3515ba0691c4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/319573b947f445aea15b0550ae111dc7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/146c9d144d7c428d8d2925209b0c9e6b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7a78084cba04471aaee1c07ac50d2365, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/3dc7300a00b848f9ba1c1a56a95410b5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/640e22bc3f01451fb15e048958f75f81, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/338d1f2aa3e4414081d924709155c885, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/742a330d6694460dabf4406717a0ce95, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6c0ce872ed1b456a837de06e48910289, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b75687188fc44925bc18c5ed06b3a214, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b021f9895e61435484c50ba2de5d8d46, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d76fbc1902694a28a8701f0de508bafd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/cd0fdf2a2665478f891ac04a614c8e37, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/c06f41ea99574e8dba9f35d4be2b69a3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b186319d41664fe482e09042d3a3737a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/85b0e85f82f14bbba983db0aa7e22191, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/a2769fb6b9104d9eaf6af7c3f54deea9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ea1b0296f77043939ca4ba7b9394287e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/87ffe6a0daff41c1bd3bbaab4970d2b4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/89dd8a25a3404fe99b47e3adfe1c6862, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6191d9ef7e7940e3bcd9851632d80807, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/aa8c5e5d16c94eb6aac2c2318b5795b1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d9e24ff7c45b49318de7b40d6cee2d30, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/f3ccb336701c433486ae79da07f05415, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/026b586c03b842c48f353c7e7d47e0d5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4dafaedd07a44b88b4294d5aff9723b0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7927b5d907904719862bfd4b8c60d659, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ca117a61aab04905a7e1eff86b6841c4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4beea6105c3f420c8f5e304ce29649f0] to archive 2024-11-26T11:16:08,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:16:08,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/e28f51417c9545baaa5a3515ba0691c4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/e28f51417c9545baaa5a3515ba0691c4 2024-11-26T11:16:08,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/319573b947f445aea15b0550ae111dc7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/319573b947f445aea15b0550ae111dc7 2024-11-26T11:16:08,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/146c9d144d7c428d8d2925209b0c9e6b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/146c9d144d7c428d8d2925209b0c9e6b 2024-11-26T11:16:08,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7a78084cba04471aaee1c07ac50d2365 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7a78084cba04471aaee1c07ac50d2365 2024-11-26T11:16:08,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/3dc7300a00b848f9ba1c1a56a95410b5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/3dc7300a00b848f9ba1c1a56a95410b5 2024-11-26T11:16:08,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/640e22bc3f01451fb15e048958f75f81 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/640e22bc3f01451fb15e048958f75f81 2024-11-26T11:16:08,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/338d1f2aa3e4414081d924709155c885 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/338d1f2aa3e4414081d924709155c885 2024-11-26T11:16:08,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/742a330d6694460dabf4406717a0ce95 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/742a330d6694460dabf4406717a0ce95 2024-11-26T11:16:08,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6c0ce872ed1b456a837de06e48910289 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6c0ce872ed1b456a837de06e48910289 2024-11-26T11:16:08,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b75687188fc44925bc18c5ed06b3a214 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b75687188fc44925bc18c5ed06b3a214 2024-11-26T11:16:08,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b021f9895e61435484c50ba2de5d8d46 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b021f9895e61435484c50ba2de5d8d46 2024-11-26T11:16:08,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d76fbc1902694a28a8701f0de508bafd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d76fbc1902694a28a8701f0de508bafd 2024-11-26T11:16:08,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/cd0fdf2a2665478f891ac04a614c8e37 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/cd0fdf2a2665478f891ac04a614c8e37 2024-11-26T11:16:08,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/c06f41ea99574e8dba9f35d4be2b69a3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/c06f41ea99574e8dba9f35d4be2b69a3 2024-11-26T11:16:08,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b186319d41664fe482e09042d3a3737a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/b186319d41664fe482e09042d3a3737a 2024-11-26T11:16:08,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/85b0e85f82f14bbba983db0aa7e22191 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/85b0e85f82f14bbba983db0aa7e22191 2024-11-26T11:16:08,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/a2769fb6b9104d9eaf6af7c3f54deea9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/a2769fb6b9104d9eaf6af7c3f54deea9 2024-11-26T11:16:08,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ea1b0296f77043939ca4ba7b9394287e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ea1b0296f77043939ca4ba7b9394287e 2024-11-26T11:16:08,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/87ffe6a0daff41c1bd3bbaab4970d2b4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/87ffe6a0daff41c1bd3bbaab4970d2b4 2024-11-26T11:16:08,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/89dd8a25a3404fe99b47e3adfe1c6862 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/89dd8a25a3404fe99b47e3adfe1c6862 2024-11-26T11:16:08,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6191d9ef7e7940e3bcd9851632d80807 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/6191d9ef7e7940e3bcd9851632d80807 2024-11-26T11:16:08,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/aa8c5e5d16c94eb6aac2c2318b5795b1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/aa8c5e5d16c94eb6aac2c2318b5795b1 2024-11-26T11:16:08,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d9e24ff7c45b49318de7b40d6cee2d30 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d9e24ff7c45b49318de7b40d6cee2d30 2024-11-26T11:16:08,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/f3ccb336701c433486ae79da07f05415 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/f3ccb336701c433486ae79da07f05415 2024-11-26T11:16:08,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/026b586c03b842c48f353c7e7d47e0d5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/026b586c03b842c48f353c7e7d47e0d5 2024-11-26T11:16:08,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4dafaedd07a44b88b4294d5aff9723b0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4dafaedd07a44b88b4294d5aff9723b0 2024-11-26T11:16:08,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7927b5d907904719862bfd4b8c60d659 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/7927b5d907904719862bfd4b8c60d659 2024-11-26T11:16:08,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ca117a61aab04905a7e1eff86b6841c4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/ca117a61aab04905a7e1eff86b6841c4 2024-11-26T11:16:08,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4beea6105c3f420c8f5e304ce29649f0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/4beea6105c3f420c8f5e304ce29649f0 2024-11-26T11:16:08,359 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/recovered.edits/952.seqid, newMaxSeqId=952, maxSeqId=1 2024-11-26T11:16:08,359 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. 2024-11-26T11:16:08,360 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for de67c03f4cecf266dfeecbd54a45d9f0: Waiting for close lock at 1732619766668Running coprocessor pre-close hooks at 1732619766668Disabling compacts and flushes for region at 1732619766668Disabling writes for close at 1732619767004 (+336 ms)Obtaining lock to block concurrent updates at 1732619767004Preparing flush snapshotting stores in de67c03f4cecf266dfeecbd54a45d9f0 at 1732619767004Finished memstore snapshotting TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732619767005 (+1 ms)Flushing stores of TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0. at 1732619767005Flushing de67c03f4cecf266dfeecbd54a45d9f0/A: creating writer at 1732619767005Flushing de67c03f4cecf266dfeecbd54a45d9f0/A: appending metadata at 1732619767007 (+2 ms)Flushing de67c03f4cecf266dfeecbd54a45d9f0/A: closing flushed file at 1732619767007Flushing de67c03f4cecf266dfeecbd54a45d9f0/B: creating writer at 1732619767415 (+408 ms)Flushing de67c03f4cecf266dfeecbd54a45d9f0/B: appending metadata at 1732619767417 (+2 ms)Flushing de67c03f4cecf266dfeecbd54a45d9f0/B: closing flushed file at 1732619767417Flushing de67c03f4cecf266dfeecbd54a45d9f0/C: creating writer at 1732619767830 (+413 ms)Flushing de67c03f4cecf266dfeecbd54a45d9f0/C: appending metadata at 1732619767833 (+3 ms)Flushing de67c03f4cecf266dfeecbd54a45d9f0/C: closing flushed file at 1732619767833Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@696e1e9f: reopening flushed file at 1732619768243 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e0ec95: reopening flushed file at 1732619768249 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e70cf88: reopening flushed file at 1732619768255 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for de67c03f4cecf266dfeecbd54a45d9f0 in 1258ms, sequenceid=949, compaction requested=true at 1732619768262 (+7 ms)Writing region close event to WAL at 1732619768356 (+94 ms)Running coprocessor post-close hooks at 1732619768359 (+3 ms)Closed at 1732619768359 2024-11-26T11:16:08,361 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:08,362 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=de67c03f4cecf266dfeecbd54a45d9f0, regionState=CLOSED 2024-11-26T11:16:08,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:16:08,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T11:16:08,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure de67c03f4cecf266dfeecbd54a45d9f0, server=7f3a3edaecc3,37499,1732619676667 in 1.8490 sec 2024-11-26T11:16:08,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-26T11:16:08,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de67c03f4cecf266dfeecbd54a45d9f0, UNASSIGN in 1.8540 sec 2024-11-26T11:16:08,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-26T11:16:08,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8570 sec 2024-11-26T11:16:08,369 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619768369"}]},"ts":"1732619768369"} 2024-11-26T11:16:08,370 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T11:16:08,370 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T11:16:08,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8650 sec 2024-11-26T11:16:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T11:16:08,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:08,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T11:16:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,655 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T11:16:08,656 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,657 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:08,659 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/recovered.edits] 2024-11-26T11:16:08,661 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/23b986ba56a145439ce86ecc6f6c869f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/23b986ba56a145439ce86ecc6f6c869f 2024-11-26T11:16:08,662 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3bb46b232f0f4effa5a01be596535fea to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/3bb46b232f0f4effa5a01be596535fea 2024-11-26T11:16:08,663 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/727b31045c664f8cbd8204ae7003d467 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/727b31045c664f8cbd8204ae7003d467 2024-11-26T11:16:08,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eb7cb753a0d443b9afa7b4ce41c02c99 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/A/eb7cb753a0d443b9afa7b4ce41c02c99 2024-11-26T11:16:08,667 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0963aacfa06b428b84df81fb90df89c7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/0963aacfa06b428b84df81fb90df89c7 2024-11-26T11:16:08,668 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/3bf715127b72468b8091fae8e8ac0162 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/3bf715127b72468b8091fae8e8ac0162 2024-11-26T11:16:08,669 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/79c7cea568184220b00ccd6d78fde435 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/79c7cea568184220b00ccd6d78fde435 2024-11-26T11:16:08,670 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0b9d079b6ed4b4abffbeb12d0e4791a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/B/d0b9d079b6ed4b4abffbeb12d0e4791a 2024-11-26T11:16:08,671 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/25dc414e804c4a19937e1f49c23e19e6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/25dc414e804c4a19937e1f49c23e19e6 2024-11-26T11:16:08,672 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/58b2acb020144ecd9d091de67a65d1c9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/58b2acb020144ecd9d091de67a65d1c9 2024-11-26T11:16:08,673 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/9af9985091764b9594c27cb9eff73c93 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/9af9985091764b9594c27cb9eff73c93 2024-11-26T11:16:08,673 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d6db3f47f53d499d9fc15537d00e75b3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/C/d6db3f47f53d499d9fc15537d00e75b3 2024-11-26T11:16:08,675 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/recovered.edits/952.seqid to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0/recovered.edits/952.seqid 2024-11-26T11:16:08,676 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/de67c03f4cecf266dfeecbd54a45d9f0 2024-11-26T11:16:08,676 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T11:16:08,678 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,680 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T11:16:08,682 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T11:16:08,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T11:16:08,683 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732619768683"}]},"ts":"9223372036854775807"} 2024-11-26T11:16:08,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T11:16:08,685 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => de67c03f4cecf266dfeecbd54a45d9f0, NAME => 'TestAcidGuarantees,,1732619742432.de67c03f4cecf266dfeecbd54a45d9f0.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T11:16:08,685 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T11:16:08,685 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732619768685"}]},"ts":"9223372036854775807"} 2024-11-26T11:16:08,687 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T11:16:08,687 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-26T11:16:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T11:16:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T11:16:08,763 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:08,774 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1301850378_22 at /127.0.0.1:36544 [Waiting for operation #426] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_181182078_22 at /127.0.0.1:53496 [Waiting for operation #196] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3212 (was 4088) 2024-11-26T11:16:08,786 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=3212 2024-11-26T11:16:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:16:08,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:16:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:08,790 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T11:16:08,791 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:08,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-11-26T11:16:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-26T11:16:08,791 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T11:16:08,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742142_1318 (size=1061) 2024-11-26T11:16:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-26T11:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-26T11:16:09,202 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:16:09,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742143_1319 (size=53) 2024-11-26T11:16:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-26T11:16:09,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:16:09,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4772d5b8c16134e9a403e97eb904a2d8, disabling compactions & flushes 2024-11-26T11:16:09,674 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:09,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:09,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. after waiting 0 ms 2024-11-26T11:16:09,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:09,674 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:09,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4772d5b8c16134e9a403e97eb904a2d8: Waiting for close lock at 1732619769674Disabling compacts and flushes for region at 1732619769674Disabling writes for close at 1732619769674Writing region close event to WAL at 1732619769674Closed at 1732619769674 2024-11-26T11:16:09,675 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T11:16:09,675 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732619769675"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732619769675"}]},"ts":"1732619769675"} 2024-11-26T11:16:09,677 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T11:16:09,680 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T11:16:09,681 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619769680"}]},"ts":"1732619769680"} 2024-11-26T11:16:09,682 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T11:16:09,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, ASSIGN}] 2024-11-26T11:16:09,684 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, ASSIGN 2024-11-26T11:16:09,685 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, ASSIGN; state=OFFLINE, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=false 2024-11-26T11:16:09,836 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:09,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, ASSIGN because future has completed 2024-11-26T11:16:09,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:16:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-26T11:16:09,993 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:09,993 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:16:09,993 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:09,993 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:16:09,993 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:09,993 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:09,995 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:09,996 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:09,996 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4772d5b8c16134e9a403e97eb904a2d8 columnFamilyName A 2024-11-26T11:16:09,996 DEBUG [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:09,996 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(327): Store=4772d5b8c16134e9a403e97eb904a2d8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:09,997 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:09,997 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:09,998 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4772d5b8c16134e9a403e97eb904a2d8 columnFamilyName B 2024-11-26T11:16:09,998 DEBUG [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:09,998 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(327): Store=4772d5b8c16134e9a403e97eb904a2d8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:09,998 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,002 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:10,002 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4772d5b8c16134e9a403e97eb904a2d8 columnFamilyName C 2024-11-26T11:16:10,002 DEBUG [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:10,003 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(327): Store=4772d5b8c16134e9a403e97eb904a2d8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:10,003 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:10,003 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,004 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,004 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,006 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,006 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,006 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:16:10,008 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,011 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:16:10,011 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 4772d5b8c16134e9a403e97eb904a2d8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59964576, jitterRate=-0.10645818710327148}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:16:10,011 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:10,012 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 4772d5b8c16134e9a403e97eb904a2d8: Running coprocessor pre-open hook at 1732619769993Writing region info on filesystem at 1732619769993Initializing all the Stores at 1732619769994 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619769994Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619769994Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619769994Cleaning up temporary data from old regions at 1732619770006 (+12 ms)Running coprocessor post-open hooks at 1732619770011 (+5 ms)Region opened successfully at 1732619770012 (+1 ms) 2024-11-26T11:16:10,013 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., pid=79, masterSystemTime=1732619769990 2024-11-26T11:16:10,015 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:10,015 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:10,016 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:10,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:16:10,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T11:16:10,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 in 179 msec 2024-11-26T11:16:10,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-26T11:16:10,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, ASSIGN in 336 msec 2024-11-26T11:16:10,022 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T11:16:10,022 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619770022"}]},"ts":"1732619770022"} 2024-11-26T11:16:10,023 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T11:16:10,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T11:16:10,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2360 sec 2024-11-26T11:16:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-26T11:16:10,943 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:10,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:10,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:10,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:10,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749ba631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:10,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d551fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:10,948 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:10,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:10,951 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:16:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:16:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@709d4953 2024-11-26T11:16:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:16:10,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:16:10,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:16:10,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:10,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742144_1320 (size=1097) 2024-11-26T11:16:11,369 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T11:16:11,369 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T11:16:11,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:16:11,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, REOPEN/MOVE}] 2024-11-26T11:16:11,378 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, REOPEN/MOVE 2024-11-26T11:16:11,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:11,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, REOPEN/MOVE because future has completed 2024-11-26T11:16:11,381 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:16:11,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:16:11,534 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,534 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:16:11,534 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 4772d5b8c16134e9a403e97eb904a2d8, disabling compactions & flushes 2024-11-26T11:16:11,534 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,534 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,534 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. after waiting 0 ms 2024-11-26T11:16:11,534 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,538 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T11:16:11,538 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,539 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 4772d5b8c16134e9a403e97eb904a2d8: Waiting for close lock at 1732619771534Running coprocessor pre-close hooks at 1732619771534Disabling compacts and flushes for region at 1732619771534Disabling writes for close at 1732619771534Writing region close event to WAL at 1732619771535 (+1 ms)Running coprocessor post-close hooks at 1732619771538 (+3 ms)Closed at 1732619771538 2024-11-26T11:16:11,539 WARN [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: 4772d5b8c16134e9a403e97eb904a2d8 to self. 2024-11-26T11:16:11,540 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,541 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=CLOSED 2024-11-26T11:16:11,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:16:11,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-26T11:16:11,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 in 162 msec 2024-11-26T11:16:11,545 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, REOPEN/MOVE; state=CLOSED, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=true 2024-11-26T11:16:11,696 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:11,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, REOPEN/MOVE because future has completed 2024-11-26T11:16:11,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:16:11,853 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,853 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:16:11,853 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,853 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:16:11,853 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,853 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,855 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,855 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:11,856 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4772d5b8c16134e9a403e97eb904a2d8 columnFamilyName A 2024-11-26T11:16:11,857 DEBUG [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:11,857 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(327): Store=4772d5b8c16134e9a403e97eb904a2d8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:11,858 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,859 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:11,859 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4772d5b8c16134e9a403e97eb904a2d8 columnFamilyName B 2024-11-26T11:16:11,859 DEBUG [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:11,859 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(327): Store=4772d5b8c16134e9a403e97eb904a2d8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:11,859 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,860 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:11,860 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4772d5b8c16134e9a403e97eb904a2d8 columnFamilyName C 2024-11-26T11:16:11,860 DEBUG [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:11,860 INFO [StoreOpener-4772d5b8c16134e9a403e97eb904a2d8-1 {}] regionserver.HStore(327): Store=4772d5b8c16134e9a403e97eb904a2d8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:11,860 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,860 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,861 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,862 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,863 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,863 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,863 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:16:11,864 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,864 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened 4772d5b8c16134e9a403e97eb904a2d8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73644687, jitterRate=0.09739135205745697}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:16:11,864 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,865 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for 4772d5b8c16134e9a403e97eb904a2d8: Running coprocessor pre-open hook at 1732619771853Writing region info on filesystem at 1732619771853Initializing all the Stores at 1732619771854 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619771854Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619771854Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619771854Cleaning up temporary data from old regions at 1732619771863 (+9 ms)Running coprocessor post-open hooks at 1732619771864 (+1 ms)Region opened successfully at 1732619771865 (+1 ms) 2024-11-26T11:16:11,866 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., pid=84, masterSystemTime=1732619771849 2024-11-26T11:16:11,868 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,868 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:11,868 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=OPEN, openSeqNum=5, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:11,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:16:11,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-11-26T11:16:11,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 in 172 msec 2024-11-26T11:16:11,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-26T11:16:11,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, REOPEN/MOVE in 495 msec 2024-11-26T11:16:11,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-26T11:16:11,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-26T11:16:11,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 920 msec 2024-11-26T11:16:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T11:16:11,878 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:11,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27819e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,881 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b7a04a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,883 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b74db6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,884 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48aa448e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8ee0f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,887 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e55fcd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,889 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69beba0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,890 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f001fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,892 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e5836b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,894 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b17aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a384f8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586a10ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227b3c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a3e968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521f29c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d935696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@316eeadb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102a21a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:11,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:11,910 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d128de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:11,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ee61db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:11,913 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,913 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,913 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,928 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,928 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,929 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,930 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:11,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:11,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,933 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-26T11:16:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,935 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T11:16:11,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:11,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:11,940 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,960 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,963 DEBUG [Thread-1470 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,963 DEBUG [Thread-1470 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,972 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:11,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:11,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:16:11,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bf1eaee2abf14c089c53cb3aec03cfd8_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619771929/Put/seqid=0 2024-11-26T11:16:11,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619771960, max timestamp=1732619771977Num uniques 859; 2024-11-26T11:16:11,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:11,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619771960, max timestamp=1732619771978Num uniques 859; 2024-11-26T11:16:11,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:11,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619771960, max timestamp=1732619771978Num uniques 859; 2024-11-26T11:16:11,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:12,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742145_1321 (size=14594) 2024-11-26T11:16:12,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619832007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619832013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619832014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619832015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619832016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T11:16:12,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619832123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619832133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619832133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619832133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619832133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T11:16:12,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619832343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619832343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619832343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619832343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619832343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,406 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:12,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bf1eaee2abf14c089c53cb3aec03cfd8_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bf1eaee2abf14c089c53cb3aec03cfd8_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:12,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/44b49e79925547dd90a39d634942661d, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:12,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/44b49e79925547dd90a39d634942661d is 175, key is test_row_0/A:col10/1732619771929/Put/seqid=0 2024-11-26T11:16:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742146_1322 (size=39549) 2024-11-26T11:16:12,419 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/44b49e79925547dd90a39d634942661d 2024-11-26T11:16:12,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/be162e0fbfce4765a43ad2434f0f5c26 is 50, key is test_row_0/B:col10/1732619771929/Put/seqid=0 2024-11-26T11:16:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742147_1323 (size=12001) 2024-11-26T11:16:12,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/be162e0fbfce4765a43ad2434f0f5c26 2024-11-26T11:16:12,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/795cdbae14c74a658180144892d99544 is 50, key is test_row_0/C:col10/1732619771929/Put/seqid=0 2024-11-26T11:16:12,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742148_1324 (size=12001) 2024-11-26T11:16:12,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:12,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T11:16:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619832662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619832663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619832663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619832663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619832663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:12,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:12,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:12,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:12,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:12,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:12,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:12,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/795cdbae14c74a658180144892d99544 2024-11-26T11:16:12,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/44b49e79925547dd90a39d634942661d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d 2024-11-26T11:16:12,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d, entries=200, sequenceid=14, filesize=38.6 K 2024-11-26T11:16:12,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/be162e0fbfce4765a43ad2434f0f5c26 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/be162e0fbfce4765a43ad2434f0f5c26 2024-11-26T11:16:12,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/be162e0fbfce4765a43ad2434f0f5c26, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T11:16:12,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/795cdbae14c74a658180144892d99544 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/795cdbae14c74a658180144892d99544 2024-11-26T11:16:12,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/795cdbae14c74a658180144892d99544, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T11:16:12,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4772d5b8c16134e9a403e97eb904a2d8 in 966ms, sequenceid=14, compaction requested=false 2024-11-26T11:16:12,897 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T11:16:12,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:13,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T11:16:13,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:13,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T11:16:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e3e8157634d946e59cb89b54a5799851_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619772003/Put/seqid=0 2024-11-26T11:16:13,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742149_1325 (size=12154) 2024-11-26T11:16:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T11:16:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:13,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619833184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619833193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619833193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619833194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619833194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619833302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619833302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619833303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619833303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619833303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:13,424 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e3e8157634d946e59cb89b54a5799851_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e3e8157634d946e59cb89b54a5799851_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e20c5fbd940341f4b31ec615e1515e9f, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e20c5fbd940341f4b31ec615e1515e9f is 175, key is test_row_0/A:col10/1732619772003/Put/seqid=0 2024-11-26T11:16:13,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742150_1326 (size=30955) 2024-11-26T11:16:13,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619833515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619833515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619833515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619833515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619833523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,829 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e20c5fbd940341f4b31ec615e1515e9f 2024-11-26T11:16:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/64719fd4131042679d834d427f15e225 is 50, key is test_row_0/B:col10/1732619772003/Put/seqid=0 2024-11-26T11:16:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619833832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619833832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619833833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619833833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742151_1327 (size=12001) 2024-11-26T11:16:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/64719fd4131042679d834d427f15e225 2024-11-26T11:16:13,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/412061ed7641495e99636486024cbee4 is 50, key is test_row_0/C:col10/1732619772003/Put/seqid=0 2024-11-26T11:16:13,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742152_1328 (size=12001) 2024-11-26T11:16:13,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619833843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:13,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:13,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:13,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/412061ed7641495e99636486024cbee4 2024-11-26T11:16:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e20c5fbd940341f4b31ec615e1515e9f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f 2024-11-26T11:16:13,859 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f, entries=150, sequenceid=50, filesize=30.2 K 2024-11-26T11:16:13,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/64719fd4131042679d834d427f15e225 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/64719fd4131042679d834d427f15e225 2024-11-26T11:16:13,863 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/64719fd4131042679d834d427f15e225, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T11:16:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/412061ed7641495e99636486024cbee4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/412061ed7641495e99636486024cbee4 2024-11-26T11:16:13,867 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/412061ed7641495e99636486024cbee4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-26T11:16:13,868 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 4772d5b8c16134e9a403e97eb904a2d8 in 857ms, sequenceid=50, compaction requested=false 2024-11-26T11:16:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-26T11:16:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-26T11:16:13,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-26T11:16:13,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-11-26T11:16:13,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 1.9380 sec 2024-11-26T11:16:13,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:16:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T11:16:14,093 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:14,094 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-26T11:16:14,095 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T11:16:14,096 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:14,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T11:16:14,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-26T11:16:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:14,248 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T11:16:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:14,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cbcd25258f44404da7cca5d2a1131e85_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619773180/Put/seqid=0 2024-11-26T11:16:14,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742153_1329 (size=12154) 2024-11-26T11:16:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:14,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619834375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619834376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619834379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619774373, max timestamp=1732619774387Num uniques 859; 2024-11-26T11:16:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619774373, max timestamp=1732619774387Num uniques 859; 2024-11-26T11:16:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619774373, max timestamp=1732619774387Num uniques 859; 2024-11-26T11:16:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T11:16:14,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619834466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619834467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619834493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619834493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619834493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619834583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619834592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:14,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cbcd25258f44404da7cca5d2a1131e85_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cbcd25258f44404da7cca5d2a1131e85_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:14,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4da0d751c10243f0b0818f1ae7405694, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4da0d751c10243f0b0818f1ae7405694 is 175, key is test_row_0/A:col10/1732619773180/Put/seqid=0 2024-11-26T11:16:14,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742154_1330 (size=30955) 2024-11-26T11:16:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T11:16:14,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619834802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619834813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619835032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619835033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619835033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,119 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4da0d751c10243f0b0818f1ae7405694 2024-11-26T11:16:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619835112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/e482a949c0ef44d2adc1f6bd05f57113 is 50, key is test_row_0/B:col10/1732619773180/Put/seqid=0 2024-11-26T11:16:15,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742155_1331 (size=12001) 2024-11-26T11:16:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619835132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T11:16:15,529 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/e482a949c0ef44d2adc1f6bd05f57113 2024-11-26T11:16:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619835543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619835553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619835553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/1201a8bda7ad4d1f8e015d0e6fb8b559 is 50, key is test_row_0/C:col10/1732619773180/Put/seqid=0 2024-11-26T11:16:15,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742156_1332 (size=12001) 2024-11-26T11:16:15,572 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/1201a8bda7ad4d1f8e015d0e6fb8b559 2024-11-26T11:16:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4da0d751c10243f0b0818f1ae7405694 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694 2024-11-26T11:16:15,581 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694, entries=150, sequenceid=61, filesize=30.2 K 2024-11-26T11:16:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/e482a949c0ef44d2adc1f6bd05f57113 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e482a949c0ef44d2adc1f6bd05f57113 2024-11-26T11:16:15,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e482a949c0ef44d2adc1f6bd05f57113, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T11:16:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/1201a8bda7ad4d1f8e015d0e6fb8b559 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/1201a8bda7ad4d1f8e015d0e6fb8b559 2024-11-26T11:16:15,595 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/1201a8bda7ad4d1f8e015d0e6fb8b559, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T11:16:15,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4772d5b8c16134e9a403e97eb904a2d8 in 1348ms, sequenceid=61, compaction requested=true 2024-11-26T11:16:15,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:15,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:15,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-26T11:16:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-26T11:16:15,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-26T11:16:15,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-26T11:16:15,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.5060 sec 2024-11-26T11:16:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:15,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T11:16:15,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:15,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:15,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:15,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:15,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:15,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:15,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126901ee90e21a3457fb85dc49e390a0668_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:15,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742157_1333 (size=14594) 2024-11-26T11:16:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619835668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619835669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619835784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619835785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619836003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619836013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,053 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:16,056 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126901ee90e21a3457fb85dc49e390a0668_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126901ee90e21a3457fb85dc49e390a0668_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:16,057 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4b856bdbcf444416b7b02c33608d3f4f, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:16,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4b856bdbcf444416b7b02c33608d3f4f is 175, key is test_row_0/A:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:16,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742158_1334 (size=39549) 2024-11-26T11:16:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T11:16:16,243 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:16,243 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-26T11:16:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:16,245 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:16,246 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:16,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:16,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619836324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619836333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:16,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4b856bdbcf444416b7b02c33608d3f4f 2024-11-26T11:16:16,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/4d0c7dc3eb0342cab1e088613e3eb1e3 is 50, key is test_row_0/B:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:16,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742159_1335 (size=12001) 2024-11-26T11:16:16,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619836554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,564 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619836573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619836573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,582 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,583 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:16,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619836843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619836843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:16,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:16,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:16,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/4d0c7dc3eb0342cab1e088613e3eb1e3 2024-11-26T11:16:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:16,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/04535f1d1d174d00880e2fa22364105b is 50, key is test_row_0/C:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:16,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742160_1336 (size=12001) 2024-11-26T11:16:17,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:17,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:17,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:17,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:17,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:17,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:17,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:17,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:17,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/04535f1d1d174d00880e2fa22364105b 2024-11-26T11:16:17,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/4b856bdbcf444416b7b02c33608d3f4f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f 2024-11-26T11:16:17,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f, entries=200, sequenceid=97, filesize=38.6 K 2024-11-26T11:16:17,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/4d0c7dc3eb0342cab1e088613e3eb1e3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4d0c7dc3eb0342cab1e088613e3eb1e3 2024-11-26T11:16:17,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4d0c7dc3eb0342cab1e088613e3eb1e3, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T11:16:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/04535f1d1d174d00880e2fa22364105b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/04535f1d1d174d00880e2fa22364105b 2024-11-26T11:16:17,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/04535f1d1d174d00880e2fa22364105b, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T11:16:17,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4772d5b8c16134e9a403e97eb904a2d8 in 1657ms, sequenceid=97, compaction requested=true 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:17,300 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:17,300 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:17,302 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:17,302 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:17,303 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:17,303 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:17,303 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,303 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,303 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/be162e0fbfce4765a43ad2434f0f5c26, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/64719fd4131042679d834d427f15e225, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e482a949c0ef44d2adc1f6bd05f57113, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4d0c7dc3eb0342cab1e088613e3eb1e3] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=46.9 K 2024-11-26T11:16:17,303 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=137.7 K 2024-11-26T11:16:17,303 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,303 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f] 2024-11-26T11:16:17,303 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting be162e0fbfce4765a43ad2434f0f5c26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619771920 2024-11-26T11:16:17,303 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b49e79925547dd90a39d634942661d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619771920 2024-11-26T11:16:17,303 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 64719fd4131042679d834d427f15e225, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732619771997 2024-11-26T11:16:17,304 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20c5fbd940341f4b31ec615e1515e9f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732619771997 2024-11-26T11:16:17,304 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e482a949c0ef44d2adc1f6bd05f57113, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732619773180 2024-11-26T11:16:17,304 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0c7dc3eb0342cab1e088613e3eb1e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619774432 2024-11-26T11:16:17,304 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4da0d751c10243f0b0818f1ae7405694, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732619773180 2024-11-26T11:16:17,304 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b856bdbcf444416b7b02c33608d3f4f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619774432 2024-11-26T11:16:17,310 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:17,311 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:17,312 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/e4ff98d323e748f7a3eae8d9e2a1903b is 50, key is test_row_0/B:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:17,315 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126236fa6fbff7b418597649ad9a0651409_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:17,318 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126236fa6fbff7b418597649ad9a0651409_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:17,318 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126236fa6fbff7b418597649ad9a0651409_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:17,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T11:16:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,320 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:16:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:17,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742161_1337 (size=12139) 2024-11-26T11:16:17,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742162_1338 (size=4469) 2024-11-26T11:16:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d4cda46ad2e94d24b24d42b1f612d56f_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619775646/Put/seqid=0 2024-11-26T11:16:17,340 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#286 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:17,341 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/ecb640354f614710bcd985788b286f75 is 175, key is test_row_0/A:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742164_1340 (size=31093) 2024-11-26T11:16:17,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742163_1339 (size=12154) 2024-11-26T11:16:17,359 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/ecb640354f614710bcd985788b286f75 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ecb640354f614710bcd985788b286f75 2024-11-26T11:16:17,363 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into ecb640354f614710bcd985788b286f75(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:17,363 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:17,363 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=12, startTime=1732619777300; duration=0sec 2024-11-26T11:16:17,363 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:17,363 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:17,363 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:17,364 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:17,364 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:17,364 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:17,364 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/795cdbae14c74a658180144892d99544, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/412061ed7641495e99636486024cbee4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/1201a8bda7ad4d1f8e015d0e6fb8b559, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/04535f1d1d174d00880e2fa22364105b] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=46.9 K 2024-11-26T11:16:17,364 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 795cdbae14c74a658180144892d99544, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619771920 2024-11-26T11:16:17,364 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412061ed7641495e99636486024cbee4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732619771997 2024-11-26T11:16:17,365 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1201a8bda7ad4d1f8e015d0e6fb8b559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732619773180 2024-11-26T11:16:17,365 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04535f1d1d174d00880e2fa22364105b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619774432 2024-11-26T11:16:17,372 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:17,372 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/25f474373e8b4aaebbd768a093a4f9f3 is 50, key is test_row_0/C:col10/1732619774455/Put/seqid=0 2024-11-26T11:16:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742165_1341 (size=12139) 2024-11-26T11:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:17,731 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/e4ff98d323e748f7a3eae8d9e2a1903b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e4ff98d323e748f7a3eae8d9e2a1903b 2024-11-26T11:16:17,735 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into e4ff98d323e748f7a3eae8d9e2a1903b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:17,735 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:17,735 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=12, startTime=1732619777300; duration=0sec 2024-11-26T11:16:17,735 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:17,735 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:17,760 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d4cda46ad2e94d24b24d42b1f612d56f_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d4cda46ad2e94d24b24d42b1f612d56f_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a0e3f000b9b54fc080845e385c112d84, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a0e3f000b9b54fc080845e385c112d84 is 175, key is test_row_0/A:col10/1732619775646/Put/seqid=0 2024-11-26T11:16:17,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742166_1342 (size=30955) 2024-11-26T11:16:17,782 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/25f474373e8b4aaebbd768a093a4f9f3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/25f474373e8b4aaebbd768a093a4f9f3 2024-11-26T11:16:17,786 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into 25f474373e8b4aaebbd768a093a4f9f3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:17,786 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:17,786 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=12, startTime=1732619777300; duration=0sec 2024-11-26T11:16:17,786 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:17,786 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:17,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:17,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619777902, max timestamp=1732619777919Num uniques 859; 2024-11-26T11:16:17,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:17,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619777902, max timestamp=1732619777919Num uniques 859; 2024-11-26T11:16:17,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:17,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619777902, max timestamp=1732619777919Num uniques 859; 2024-11-26T11:16:17,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619838005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619838006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619838113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619838123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,165 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a0e3f000b9b54fc080845e385c112d84 2024-11-26T11:16:18,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/b1cce69c15364375944f7fbb8d7f39a6 is 50, key is test_row_0/B:col10/1732619775646/Put/seqid=0 2024-11-26T11:16:18,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742167_1343 (size=12001) 2024-11-26T11:16:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619838323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619838342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:18,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/b1cce69c15364375944f7fbb8d7f39a6 2024-11-26T11:16:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619838583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,593 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/28b0b22c51fd4b5a985cacda7a890a69 is 50, key is test_row_0/C:col10/1732619775646/Put/seqid=0 2024-11-26T11:16:18,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742168_1344 (size=12001) 2024-11-26T11:16:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619838592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619838592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,602 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:18,602 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619838644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619838653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,000 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/28b0b22c51fd4b5a985cacda7a890a69 2024-11-26T11:16:19,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a0e3f000b9b54fc080845e385c112d84 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84 2024-11-26T11:16:19,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84, entries=150, sequenceid=107, filesize=30.2 K 2024-11-26T11:16:19,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/b1cce69c15364375944f7fbb8d7f39a6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/b1cce69c15364375944f7fbb8d7f39a6 2024-11-26T11:16:19,010 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/b1cce69c15364375944f7fbb8d7f39a6, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T11:16:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/28b0b22c51fd4b5a985cacda7a890a69 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/28b0b22c51fd4b5a985cacda7a890a69 2024-11-26T11:16:19,014 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/28b0b22c51fd4b5a985cacda7a890a69, entries=150, sequenceid=107, filesize=11.7 K 2024-11-26T11:16:19,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4772d5b8c16134e9a403e97eb904a2d8 in 1695ms, sequenceid=107, compaction requested=false 2024-11-26T11:16:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-26T11:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-26T11:16:19,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-26T11:16:19,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7700 sec 2024-11-26T11:16:19,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.7750 sec 2024-11-26T11:16:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:19,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T11:16:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:19,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261af0b2c015d44056897a2f2ac0d5e863_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:19,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742169_1345 (size=14794) 2024-11-26T11:16:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619839198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619839206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619839312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619839313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619839523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619839532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:19,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261af0b2c015d44056897a2f2ac0d5e863_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261af0b2c015d44056897a2f2ac0d5e863_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:19,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/b03497471241478b8976b50d8c13810c, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:19,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/b03497471241478b8976b50d8c13810c is 175, key is test_row_0/A:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:19,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742170_1346 (size=39749) 2024-11-26T11:16:19,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619839833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619839852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:19,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/b03497471241478b8976b50d8c13810c 2024-11-26T11:16:19,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/7264ada4ae2648c0bf687222c20c4bc2 is 50, key is test_row_0/B:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:20,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742171_1347 (size=12151) 2024-11-26T11:16:20,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619840353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:20,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619840363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:20,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/7264ada4ae2648c0bf687222c20c4bc2 2024-11-26T11:16:20,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/2ea873ce9955498e93eac290443617f7 is 50, key is test_row_0/C:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T11:16:20,413 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:20,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-26T11:16:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T11:16:20,415 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:20,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742172_1348 (size=12151) 2024-11-26T11:16:20,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:20,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T11:16:20,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T11:16:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:20,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T11:16:20,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:20,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:20,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T11:16:20,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/2ea873ce9955498e93eac290443617f7 2024-11-26T11:16:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/b03497471241478b8976b50d8c13810c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c 2024-11-26T11:16:20,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c, entries=200, sequenceid=147, filesize=38.8 K 2024-11-26T11:16:20,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/7264ada4ae2648c0bf687222c20c4bc2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/7264ada4ae2648c0bf687222c20c4bc2 2024-11-26T11:16:20,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/7264ada4ae2648c0bf687222c20c4bc2, entries=150, sequenceid=147, filesize=11.9 K 2024-11-26T11:16:20,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/2ea873ce9955498e93eac290443617f7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/2ea873ce9955498e93eac290443617f7 2024-11-26T11:16:20,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/2ea873ce9955498e93eac290443617f7, entries=150, sequenceid=147, filesize=11.9 K 2024-11-26T11:16:20,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4772d5b8c16134e9a403e97eb904a2d8 in 1662ms, sequenceid=147, compaction requested=true 2024-11-26T11:16:20,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:20,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:20,832 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:20,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:20,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:20,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:20,833 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:20,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:20,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:20,833 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:20,833 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:20,833 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:20,833 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:20,833 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,833 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,833 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e4ff98d323e748f7a3eae8d9e2a1903b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/b1cce69c15364375944f7fbb8d7f39a6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/7264ada4ae2648c0bf687222c20c4bc2] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=35.4 K 2024-11-26T11:16:20,833 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ecb640354f614710bcd985788b286f75, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=99.4 K 2024-11-26T11:16:20,833 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,833 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ecb640354f614710bcd985788b286f75, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c] 2024-11-26T11:16:20,834 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ff98d323e748f7a3eae8d9e2a1903b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619774432 2024-11-26T11:16:20,834 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb640354f614710bcd985788b286f75, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619774432 2024-11-26T11:16:20,834 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cce69c15364375944f7fbb8d7f39a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732619775646 2024-11-26T11:16:20,834 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0e3f000b9b54fc080845e385c112d84, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732619775646 2024-11-26T11:16:20,834 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7264ada4ae2648c0bf687222c20c4bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732619778003 2024-11-26T11:16:20,835 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b03497471241478b8976b50d8c13810c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732619777994 2024-11-26T11:16:20,840 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:20,840 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/fae3b364b4cd4093b55de496d6cc26a7 is 50, key is test_row_0/B:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:20,841 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:20,846 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112616c4514db05242f69c330a3d25f9f779_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:20,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742173_1349 (size=12391) 2024-11-26T11:16:20,849 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112616c4514db05242f69c330a3d25f9f779_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:20,849 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112616c4514db05242f69c330a3d25f9f779_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:20,852 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/fae3b364b4cd4093b55de496d6cc26a7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/fae3b364b4cd4093b55de496d6cc26a7 2024-11-26T11:16:20,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742174_1350 (size=4469) 2024-11-26T11:16:20,858 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into fae3b364b4cd4093b55de496d6cc26a7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:20,858 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:20,858 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=13, startTime=1732619780832; duration=0sec 2024-11-26T11:16:20,858 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:20,858 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:20,858 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:20,859 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:20,859 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:20,859 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,859 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/25f474373e8b4aaebbd768a093a4f9f3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/28b0b22c51fd4b5a985cacda7a890a69, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/2ea873ce9955498e93eac290443617f7] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=35.4 K 2024-11-26T11:16:20,859 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f474373e8b4aaebbd768a093a4f9f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732619774432 2024-11-26T11:16:20,860 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 28b0b22c51fd4b5a985cacda7a890a69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732619775646 2024-11-26T11:16:20,860 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea873ce9955498e93eac290443617f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732619778003 2024-11-26T11:16:20,865 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:20,865 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/186b6ff717ad4a98a9ec54e912748831 is 50, key is test_row_0/C:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742175_1351 (size=12391) 2024-11-26T11:16:20,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:20,874 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260e09aacc4e4e4e5a854ef20f373e7c3e_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619779197/Put/seqid=0 2024-11-26T11:16:20,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742176_1352 (size=12304) 2024-11-26T11:16:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T11:16:21,255 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:21,255 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/612732afa30940ab80c2c847f42bc887 is 175, key is test_row_0/A:col10/1732619778004/Put/seqid=0 2024-11-26T11:16:21,274 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/186b6ff717ad4a98a9ec54e912748831 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/186b6ff717ad4a98a9ec54e912748831 2024-11-26T11:16:21,280 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into 186b6ff717ad4a98a9ec54e912748831(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:21,280 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:21,280 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=13, startTime=1732619780833; duration=0sec 2024-11-26T11:16:21,280 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:21,280 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:21,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:21,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260e09aacc4e4e4e5a854ef20f373e7c3e_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e09aacc4e4e4e5a854ef20f373e7c3e_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:21,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e771a82984d74c239ef8c4f0e0972e10, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e771a82984d74c239ef8c4f0e0972e10 is 175, key is test_row_0/A:col10/1732619779197/Put/seqid=0 2024-11-26T11:16:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742177_1353 (size=31345) 2024-11-26T11:16:21,318 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/612732afa30940ab80c2c847f42bc887 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/612732afa30940ab80c2c847f42bc887 2024-11-26T11:16:21,324 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into 612732afa30940ab80c2c847f42bc887(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:21,324 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:21,324 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=13, startTime=1732619780832; duration=0sec 2024-11-26T11:16:21,324 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:21,324 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742178_1354 (size=31105) 2024-11-26T11:16:21,353 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e771a82984d74c239ef8c4f0e0972e10 2024-11-26T11:16:21,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/c08ea95439ac4dd9843630ef7d0131e9 is 50, key is test_row_0/B:col10/1732619779197/Put/seqid=0 2024-11-26T11:16:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:21,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:21,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742179_1355 (size=12151) 2024-11-26T11:16:21,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/c08ea95439ac4dd9843630ef7d0131e9 2024-11-26T11:16:21,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/ee5b3016e9e943438b95a6ff45d62da2 is 50, key is test_row_0/C:col10/1732619779197/Put/seqid=0 2024-11-26T11:16:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742180_1356 (size=12151) 2024-11-26T11:16:21,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/ee5b3016e9e943438b95a6ff45d62da2 2024-11-26T11:16:21,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/e771a82984d74c239ef8c4f0e0972e10 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10 2024-11-26T11:16:21,480 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10, entries=150, sequenceid=157, filesize=30.4 K 2024-11-26T11:16:21,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/c08ea95439ac4dd9843630ef7d0131e9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/c08ea95439ac4dd9843630ef7d0131e9 2024-11-26T11:16:21,487 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/c08ea95439ac4dd9843630ef7d0131e9, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T11:16:21,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/ee5b3016e9e943438b95a6ff45d62da2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/ee5b3016e9e943438b95a6ff45d62da2 2024-11-26T11:16:21,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/ee5b3016e9e943438b95a6ff45d62da2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T11:16:21,500 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=100.63 KB/103050 for 4772d5b8c16134e9a403e97eb904a2d8 in 626ms, sequenceid=157, compaction requested=false 2024-11-26T11:16:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-26T11:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-26T11:16:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:21,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:16:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:21,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-26T11:16:21,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-26T11:16:21,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.0940 sec 2024-11-26T11:16:21,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fa7ae9ec98d14d6680375ca1a8e94245_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T11:16:21,554 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:21,555 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-26T11:16:21,559 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:21,560 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:21,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:21,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742181_1357 (size=14794) 2024-11-26T11:16:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619841590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:21,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619841590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:21,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T11:16:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:21,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619841713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619841713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:21,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T11:16:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:21,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:21,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619841933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619841933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:21,963 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:21,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fa7ae9ec98d14d6680375ca1a8e94245_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fa7ae9ec98d14d6680375ca1a8e94245_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:21,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/5fe049ae100f439c8717dde937601dfb, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:21,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/5fe049ae100f439c8717dde937601dfb is 175, key is test_row_0/A:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:21,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742182_1358 (size=39749) 2024-11-26T11:16:21,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/5fe049ae100f439c8717dde937601dfb 2024-11-26T11:16:21,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0bde5dac61454fc886d1f94145c6402b is 50, key is test_row_0/B:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:21,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742183_1359 (size=12151) 2024-11-26T11:16:21,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0bde5dac61454fc886d1f94145c6402b 2024-11-26T11:16:22,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/f4417afcb9694cf997275ed09f8dee8e is 50, key is test_row_0/C:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:22,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742184_1360 (size=12151) 2024-11-26T11:16:22,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T11:16:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T11:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:22,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619842253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619842253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T11:16:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:22,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/f4417afcb9694cf997275ed09f8dee8e 2024-11-26T11:16:22,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/5fe049ae100f439c8717dde937601dfb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb 2024-11-26T11:16:22,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb, entries=200, sequenceid=178, filesize=38.8 K 2024-11-26T11:16:22,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0bde5dac61454fc886d1f94145c6402b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0bde5dac61454fc886d1f94145c6402b 2024-11-26T11:16:22,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0bde5dac61454fc886d1f94145c6402b, entries=150, sequenceid=178, filesize=11.9 K 2024-11-26T11:16:22,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/f4417afcb9694cf997275ed09f8dee8e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/f4417afcb9694cf997275ed09f8dee8e 2024-11-26T11:16:22,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/f4417afcb9694cf997275ed09f8dee8e, entries=150, sequenceid=178, filesize=11.9 K 2024-11-26T11:16:22,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 4772d5b8c16134e9a403e97eb904a2d8 in 925ms, sequenceid=178, compaction requested=true 2024-11-26T11:16:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:22,431 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:22,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:22,431 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:22,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:22,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:22,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:22,431 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:22,431 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:22,431 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:22,431 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:22,432 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,432 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,432 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/612732afa30940ab80c2c847f42bc887, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=99.8 K 2024-11-26T11:16:22,432 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/fae3b364b4cd4093b55de496d6cc26a7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/c08ea95439ac4dd9843630ef7d0131e9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0bde5dac61454fc886d1f94145c6402b] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=35.8 K 2024-11-26T11:16:22,432 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,432 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/612732afa30940ab80c2c847f42bc887, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb] 2024-11-26T11:16:22,432 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting fae3b364b4cd4093b55de496d6cc26a7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732619778003 2024-11-26T11:16:22,432 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 612732afa30940ab80c2c847f42bc887, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732619778003 2024-11-26T11:16:22,432 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c08ea95439ac4dd9843630ef7d0131e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732619779191 2024-11-26T11:16:22,432 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e771a82984d74c239ef8c4f0e0972e10, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732619779191 2024-11-26T11:16:22,433 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bde5dac61454fc886d1f94145c6402b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732619781432 2024-11-26T11:16:22,433 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fe049ae100f439c8717dde937601dfb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732619781432 2024-11-26T11:16:22,439 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:22,440 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:22,441 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/13bd98f556a340d9ad4a39685048e3a1 is 50, key is test_row_0/B:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:22,441 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126ca93912225664235a12db92162d6bd88_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:22,442 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126ca93912225664235a12db92162d6bd88_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:22,442 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ca93912225664235a12db92162d6bd88_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:22,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742186_1362 (size=4469) 2024-11-26T11:16:22,451 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#304 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:22,451 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/7ef4d38348c9401bab76b7f762809509 is 175, key is test_row_0/A:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:22,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742185_1361 (size=12493) 2024-11-26T11:16:22,455 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/13bd98f556a340d9ad4a39685048e3a1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/13bd98f556a340d9ad4a39685048e3a1 2024-11-26T11:16:22,459 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into 13bd98f556a340d9ad4a39685048e3a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:22,459 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:22,459 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=13, startTime=1732619782431; duration=0sec 2024-11-26T11:16:22,459 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:22,459 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:22,459 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:22,462 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:22,462 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:22,462 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,462 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/186b6ff717ad4a98a9ec54e912748831, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/ee5b3016e9e943438b95a6ff45d62da2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/f4417afcb9694cf997275ed09f8dee8e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=35.8 K 2024-11-26T11:16:22,462 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 186b6ff717ad4a98a9ec54e912748831, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732619778003 2024-11-26T11:16:22,463 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5b3016e9e943438b95a6ff45d62da2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732619779191 2024-11-26T11:16:22,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742187_1363 (size=31447) 2024-11-26T11:16:22,463 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f4417afcb9694cf997275ed09f8dee8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732619781432 2024-11-26T11:16:22,468 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:22,468 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/3ef3449edb0a4d2f83b0718105aa2e5b is 50, key is test_row_0/C:col10/1732619781432/Put/seqid=0 2024-11-26T11:16:22,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742188_1364 (size=12493) 2024-11-26T11:16:22,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T11:16:22,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:22,481 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:16:22,481 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/3ef3449edb0a4d2f83b0718105aa2e5b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/3ef3449edb0a4d2f83b0718105aa2e5b 2024-11-26T11:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:22,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d5369c2a2811401fa491be3848fb01d9_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619781551/Put/seqid=0 2024-11-26T11:16:22,488 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into 3ef3449edb0a4d2f83b0718105aa2e5b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:22,488 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:22,488 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=13, startTime=1732619782431; duration=0sec 2024-11-26T11:16:22,489 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:22,489 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:22,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742189_1365 (size=12304) 2024-11-26T11:16:22,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:22,513 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d5369c2a2811401fa491be3848fb01d9_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d5369c2a2811401fa491be3848fb01d9_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:22,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/858c480b7fc34ba3bd2f5b743366fade, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:22,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/858c480b7fc34ba3bd2f5b743366fade is 175, key is test_row_0/A:col10/1732619781551/Put/seqid=0 2024-11-26T11:16:22,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742190_1366 (size=31105) 2024-11-26T11:16:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:22,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619842647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619842657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619842658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619842763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619842763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619842763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619842772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619842772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,868 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/7ef4d38348c9401bab76b7f762809509 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7ef4d38348c9401bab76b7f762809509 2024-11-26T11:16:22,872 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into 7ef4d38348c9401bab76b7f762809509(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:22,872 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:22,872 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=13, startTime=1732619782430; duration=0sec 2024-11-26T11:16:22,872 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:22,872 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:22,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/858c480b7fc34ba3bd2f5b743366fade 2024-11-26T11:16:22,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/88afbfee810442869a648610b7c3cb59 is 50, key is test_row_0/B:col10/1732619781551/Put/seqid=0 2024-11-26T11:16:22,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742191_1367 (size=12151) 2024-11-26T11:16:22,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619842983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619842983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:22,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619842992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619843293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619843293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619843303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/88afbfee810442869a648610b7c3cb59 2024-11-26T11:16:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d4ac54dd478648f2bd10a2333f635b87 is 50, key is test_row_0/C:col10/1732619781551/Put/seqid=0 2024-11-26T11:16:23,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742192_1368 (size=12151) 2024-11-26T11:16:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:23,745 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d4ac54dd478648f2bd10a2333f635b87 2024-11-26T11:16:23,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/858c480b7fc34ba3bd2f5b743366fade as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade 2024-11-26T11:16:23,753 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade, entries=150, sequenceid=193, filesize=30.4 K 2024-11-26T11:16:23,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/88afbfee810442869a648610b7c3cb59 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/88afbfee810442869a648610b7c3cb59 2024-11-26T11:16:23,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/88afbfee810442869a648610b7c3cb59, entries=150, sequenceid=193, filesize=11.9 K 2024-11-26T11:16:23,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d4ac54dd478648f2bd10a2333f635b87 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d4ac54dd478648f2bd10a2333f635b87 2024-11-26T11:16:23,766 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d4ac54dd478648f2bd10a2333f635b87, entries=150, sequenceid=193, filesize=11.9 K 2024-11-26T11:16:23,767 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4772d5b8c16134e9a403e97eb904a2d8 in 1286ms, sequenceid=193, compaction requested=false 2024-11-26T11:16:23,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:23,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:23,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-26T11:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-26T11:16:23,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-26T11:16:23,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2090 sec 2024-11-26T11:16:23,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2190 sec 2024-11-26T11:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:23,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c45991ed5a394394a52b69342ac4cc33_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619843822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619843823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742193_1369 (size=14794) 2024-11-26T11:16:23,829 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:23,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c45991ed5a394394a52b69342ac4cc33_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c45991ed5a394394a52b69342ac4cc33_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:23,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/89b1e75bc5ac4f9ca731088ccaff8aa8, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:23,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/89b1e75bc5ac4f9ca731088ccaff8aa8 is 175, key is test_row_0/A:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619843825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619843826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619843826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742194_1370 (size=39749) 2024-11-26T11:16:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619843935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619843944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619843944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:23,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619843944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619844153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619844162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619844163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619844163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/89b1e75bc5ac4f9ca731088ccaff8aa8 2024-11-26T11:16:24,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/68f5ab8bd4b74ed09d28b8f8c4922997 is 50, key is test_row_0/B:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:24,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742195_1371 (size=12151) 2024-11-26T11:16:24,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619844472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619844483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619844483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619844483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:24,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/68f5ab8bd4b74ed09d28b8f8c4922997 2024-11-26T11:16:24,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/db4baacf61a1438689d10a3e302119d6 is 50, key is test_row_0/C:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:24,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742196_1372 (size=12151) 2024-11-26T11:16:24,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/db4baacf61a1438689d10a3e302119d6 2024-11-26T11:16:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/89b1e75bc5ac4f9ca731088ccaff8aa8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8 2024-11-26T11:16:24,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8, entries=200, sequenceid=214, filesize=38.8 K 2024-11-26T11:16:24,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/68f5ab8bd4b74ed09d28b8f8c4922997 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/68f5ab8bd4b74ed09d28b8f8c4922997 2024-11-26T11:16:24,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/68f5ab8bd4b74ed09d28b8f8c4922997, entries=150, sequenceid=214, filesize=11.9 K 2024-11-26T11:16:24,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/db4baacf61a1438689d10a3e302119d6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/db4baacf61a1438689d10a3e302119d6 2024-11-26T11:16:24,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/db4baacf61a1438689d10a3e302119d6, entries=150, sequenceid=214, filesize=11.9 K 2024-11-26T11:16:24,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4772d5b8c16134e9a403e97eb904a2d8 in 986ms, sequenceid=214, compaction requested=true 2024-11-26T11:16:24,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:24,779 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:24,779 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:24,783 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:24,783 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:24,784 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:24,784 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:24,784 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:24,784 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7ef4d38348c9401bab76b7f762809509, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=99.9 K 2024-11-26T11:16:24,784 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:24,784 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7ef4d38348c9401bab76b7f762809509, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8] 2024-11-26T11:16:24,784 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:24,784 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/13bd98f556a340d9ad4a39685048e3a1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/88afbfee810442869a648610b7c3cb59, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/68f5ab8bd4b74ed09d28b8f8c4922997] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=35.9 K 2024-11-26T11:16:24,785 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef4d38348c9401bab76b7f762809509, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732619781432 2024-11-26T11:16:24,786 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bd98f556a340d9ad4a39685048e3a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732619781432 2024-11-26T11:16:24,786 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858c480b7fc34ba3bd2f5b743366fade, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732619781551 2024-11-26T11:16:24,790 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 88afbfee810442869a648610b7c3cb59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732619781551 2024-11-26T11:16:24,791 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b1e75bc5ac4f9ca731088ccaff8aa8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732619782619 2024-11-26T11:16:24,791 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f5ab8bd4b74ed09d28b8f8c4922997, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732619782646 2024-11-26T11:16:24,806 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:24,806 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:24,807 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/86ddc394b67141b8937061e8c4d190ea is 50, key is test_row_0/B:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:24,825 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f39d08b855cd41ba9264fefb881b22a7_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:24,827 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f39d08b855cd41ba9264fefb881b22a7_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:24,827 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f39d08b855cd41ba9264fefb881b22a7_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:24,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:16:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:24,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742197_1373 (size=12595) 2024-11-26T11:16:24,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742198_1374 (size=4469) 2024-11-26T11:16:24,851 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#313 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:24,857 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/ad049f963d644a288903e0786fa474c6 is 175, key is test_row_0/A:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:24,864 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/86ddc394b67141b8937061e8c4d190ea as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/86ddc394b67141b8937061e8c4d190ea 2024-11-26T11:16:24,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e30e688488534f2c8aad67626c4424af_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619783824/Put/seqid=0 2024-11-26T11:16:24,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742199_1375 (size=31549) 2024-11-26T11:16:24,878 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into 86ddc394b67141b8937061e8c4d190ea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:24,878 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:24,878 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=13, startTime=1732619784779; duration=0sec 2024-11-26T11:16:24,878 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:24,879 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:24,879 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:24,879 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:24,879 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:24,880 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:24,880 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/3ef3449edb0a4d2f83b0718105aa2e5b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d4ac54dd478648f2bd10a2333f635b87, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/db4baacf61a1438689d10a3e302119d6] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=35.9 K 2024-11-26T11:16:24,880 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef3449edb0a4d2f83b0718105aa2e5b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732619781432 2024-11-26T11:16:24,880 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ac54dd478648f2bd10a2333f635b87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732619781551 2024-11-26T11:16:24,881 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting db4baacf61a1438689d10a3e302119d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732619782646 2024-11-26T11:16:24,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742200_1376 (size=14794) 2024-11-26T11:16:24,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:24,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e30e688488534f2c8aad67626c4424af_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e30e688488534f2c8aad67626c4424af_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:24,901 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:24,902 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/a3ecae654b0c4e8bb8be21c06ee9686f is 50, key is test_row_0/C:col10/1732619782657/Put/seqid=0 2024-11-26T11:16:24,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6052be1f405a489a8d4515689e800f82, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:24,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6052be1f405a489a8d4515689e800f82 is 175, key is test_row_0/A:col10/1732619783824/Put/seqid=0 2024-11-26T11:16:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742201_1377 (size=12595) 2024-11-26T11:16:24,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742202_1378 (size=39749) 2024-11-26T11:16:25,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619845003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619845016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619845026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619845026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619845026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619845133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619845153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619845153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619845153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,282 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/ad049f963d644a288903e0786fa474c6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ad049f963d644a288903e0786fa474c6 2024-11-26T11:16:25,287 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into ad049f963d644a288903e0786fa474c6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:25,287 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:25,287 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=13, startTime=1732619784779; duration=0sec 2024-11-26T11:16:25,288 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:25,288 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:25,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6052be1f405a489a8d4515689e800f82 2024-11-26T11:16:25,331 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/a3ecae654b0c4e8bb8be21c06ee9686f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a3ecae654b0c4e8bb8be21c06ee9686f 2024-11-26T11:16:25,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/32e6f56e09a94eaeb493c9b5002f8b02 is 50, key is test_row_0/B:col10/1732619783824/Put/seqid=0 2024-11-26T11:16:25,341 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into a3ecae654b0c4e8bb8be21c06ee9686f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:25,341 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:25,341 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=13, startTime=1732619784779; duration=0sec 2024-11-26T11:16:25,341 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:25,341 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:25,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619845354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619845373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619845373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619845373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742203_1379 (size=12151) 2024-11-26T11:16:25,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/32e6f56e09a94eaeb493c9b5002f8b02 2024-11-26T11:16:25,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/481669eae593493281217398be6f6184 is 50, key is test_row_0/C:col10/1732619783824/Put/seqid=0 2024-11-26T11:16:25,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742204_1380 (size=12151) 2024-11-26T11:16:25,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619845683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619845683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619845683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619845683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T11:16:25,724 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:25,724 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-26T11:16:25,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:25,728 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:25,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T11:16:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T11:16:25,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/481669eae593493281217398be6f6184 2024-11-26T11:16:25,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6052be1f405a489a8d4515689e800f82 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82 2024-11-26T11:16:25,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82, entries=200, sequenceid=228, filesize=38.8 K 2024-11-26T11:16:25,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T11:16:25,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:25,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:25,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:25,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:25,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/32e6f56e09a94eaeb493c9b5002f8b02 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/32e6f56e09a94eaeb493c9b5002f8b02 2024-11-26T11:16:25,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/32e6f56e09a94eaeb493c9b5002f8b02, entries=150, sequenceid=228, filesize=11.9 K 2024-11-26T11:16:25,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/481669eae593493281217398be6f6184 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/481669eae593493281217398be6f6184 2024-11-26T11:16:25,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/481669eae593493281217398be6f6184, entries=150, sequenceid=228, filesize=11.9 K 2024-11-26T11:16:25,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 4772d5b8c16134e9a403e97eb904a2d8 in 1074ms, sequenceid=228, compaction requested=false 2024-11-26T11:16:25,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:26,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T11:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:26,036 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T11:16:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d85b56690a2a4257af4e6d4ca07f4aff_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619785019/Put/seqid=0 2024-11-26T11:16:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742205_1381 (size=12304) 2024-11-26T11:16:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:26,081 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d85b56690a2a4257af4e6d4ca07f4aff_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d85b56690a2a4257af4e6d4ca07f4aff_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:26,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a5c44b688c184a60998e202533f4f236, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:26,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a5c44b688c184a60998e202533f4f236 is 175, key is test_row_0/A:col10/1732619785019/Put/seqid=0 2024-11-26T11:16:26,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742206_1382 (size=31105) 2024-11-26T11:16:26,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619846154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619846195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619846195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619846203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619846203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619846272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T11:16:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619846483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,516 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a5c44b688c184a60998e202533f4f236 2024-11-26T11:16:26,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/d203b1e0b18042909072bc02ffd6c3cd is 50, key is test_row_0/B:col10/1732619785019/Put/seqid=0 2024-11-26T11:16:26,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742207_1383 (size=12151) 2024-11-26T11:16:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619846802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T11:16:26,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/d203b1e0b18042909072bc02ffd6c3cd 2024-11-26T11:16:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/73103b0f967248f9b937c18dcd461964 is 50, key is test_row_0/C:col10/1732619785019/Put/seqid=0 2024-11-26T11:16:26,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742208_1384 (size=12151) 2024-11-26T11:16:26,967 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/73103b0f967248f9b937c18dcd461964 2024-11-26T11:16:26,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/a5c44b688c184a60998e202533f4f236 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236 2024-11-26T11:16:26,981 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236, entries=150, sequenceid=250, filesize=30.4 K 2024-11-26T11:16:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/d203b1e0b18042909072bc02ffd6c3cd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d203b1e0b18042909072bc02ffd6c3cd 2024-11-26T11:16:26,985 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d203b1e0b18042909072bc02ffd6c3cd, entries=150, sequenceid=250, filesize=11.9 K 2024-11-26T11:16:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/73103b0f967248f9b937c18dcd461964 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/73103b0f967248f9b937c18dcd461964 2024-11-26T11:16:26,990 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/73103b0f967248f9b937c18dcd461964, entries=150, sequenceid=250, filesize=11.9 K 2024-11-26T11:16:26,991 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4772d5b8c16134e9a403e97eb904a2d8 in 954ms, sequenceid=250, compaction requested=true 2024-11-26T11:16:26,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:26,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:26,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-26T11:16:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-26T11:16:26,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-26T11:16:26,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2640 sec 2024-11-26T11:16:26,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.2690 sec 2024-11-26T11:16:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:27,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:16:27,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:27,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:27,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:27,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:27,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:27,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:27,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112611ea4d4bc02f49c9bb474c69c6b76849_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742209_1385 (size=14944) 2024-11-26T11:16:27,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:27,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112611ea4d4bc02f49c9bb474c69c6b76849_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112611ea4d4bc02f49c9bb474c69c6b76849_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:27,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/c9fbe3208f7a4eda8336cc13deaa7499, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:27,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/c9fbe3208f7a4eda8336cc13deaa7499 is 175, key is test_row_0/A:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619847275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742210_1386 (size=39899) 2024-11-26T11:16:27,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/c9fbe3208f7a4eda8336cc13deaa7499 2024-11-26T11:16:27,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619847294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619847294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619847296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/d939773f5e3f4bf28694153f240cb479 is 50, key is test_row_0/B:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619847323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742211_1387 (size=12301) 2024-11-26T11:16:27,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/d939773f5e3f4bf28694153f240cb479 2024-11-26T11:16:27,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/7146ea6e7b234ed49feeba7fe2f2abf9 is 50, key is test_row_0/C:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742212_1388 (size=12301) 2024-11-26T11:16:27,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/7146ea6e7b234ed49feeba7fe2f2abf9 2024-11-26T11:16:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/c9fbe3208f7a4eda8336cc13deaa7499 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499 2024-11-26T11:16:27,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499, entries=200, sequenceid=264, filesize=39.0 K 2024-11-26T11:16:27,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/d939773f5e3f4bf28694153f240cb479 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d939773f5e3f4bf28694153f240cb479 2024-11-26T11:16:27,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d939773f5e3f4bf28694153f240cb479, entries=150, sequenceid=264, filesize=12.0 K 2024-11-26T11:16:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/7146ea6e7b234ed49feeba7fe2f2abf9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7146ea6e7b234ed49feeba7fe2f2abf9 2024-11-26T11:16:27,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619847403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7146ea6e7b234ed49feeba7fe2f2abf9, entries=150, sequenceid=264, filesize=12.0 K 2024-11-26T11:16:27,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4772d5b8c16134e9a403e97eb904a2d8 in 206ms, sequenceid=264, compaction requested=true 2024-11-26T11:16:27,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:27,426 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:27,427 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:27,427 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:27,427 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:27,427 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ad049f963d644a288903e0786fa474c6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=139.0 K 2024-11-26T11:16:27,427 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:27,427 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ad049f963d644a288903e0786fa474c6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499] 2024-11-26T11:16:27,428 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad049f963d644a288903e0786fa474c6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732619782646 2024-11-26T11:16:27,428 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6052be1f405a489a8d4515689e800f82, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732619783823 2024-11-26T11:16:27,428 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c44b688c184a60998e202533f4f236, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732619784956 2024-11-26T11:16:27,429 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9fbe3208f7a4eda8336cc13deaa7499, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732619786051 2024-11-26T11:16:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:27,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:27,435 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:27,437 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:27,437 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:27,437 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:27,437 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/86ddc394b67141b8937061e8c4d190ea, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/32e6f56e09a94eaeb493c9b5002f8b02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d203b1e0b18042909072bc02ffd6c3cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d939773f5e3f4bf28694153f240cb479] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=48.0 K 2024-11-26T11:16:27,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:27,438 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ddc394b67141b8937061e8c4d190ea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732619782646 2024-11-26T11:16:27,438 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 32e6f56e09a94eaeb493c9b5002f8b02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732619783823 2024-11-26T11:16:27,439 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d203b1e0b18042909072bc02ffd6c3cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732619784956 2024-11-26T11:16:27,439 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d939773f5e3f4bf28694153f240cb479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732619786133 2024-11-26T11:16:27,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619787288, max timestamp=1732619787437Num uniques 859; 2024-11-26T11:16:27,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:16:27,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619787288, max timestamp=1732619787437Num uniques 859; 2024-11-26T11:16:27,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:16:27,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619787288, max timestamp=1732619787437Num uniques 859; 2024-11-26T11:16:27,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:16:27,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:27,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:27,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:27,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=40.25 KB heapSize=96.70 KB 2024-11-26T11:16:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:27,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:27,452 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:27,456 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:27,457 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/a956fac079f345ee90dba15da92b3aa3 is 50, key is test_row_0/B:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,467 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126659666a1a5b74eda902f3e0f00fbc561_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:27,470 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126659666a1a5b74eda902f3e0f00fbc561_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:27,470 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126659666a1a5b74eda902f3e0f00fbc561_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:27,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f9201f2275e45fe86c0e4960d6902d2_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619787437/Put/seqid=0 2024-11-26T11:16:27,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742213_1389 (size=12881) 2024-11-26T11:16:27,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742214_1390 (size=4469) 2024-11-26T11:16:27,535 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#324 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:27,535 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/7275b61a89db4e2fb94099fec4daa7ff is 175, key is test_row_0/A:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,540 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/a956fac079f345ee90dba15da92b3aa3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a956fac079f345ee90dba15da92b3aa3 2024-11-26T11:16:27,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742215_1391 (size=12454) 2024-11-26T11:16:27,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:27,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f9201f2275e45fe86c0e4960d6902d2_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f9201f2275e45fe86c0e4960d6902d2_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:27,545 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into a956fac079f345ee90dba15da92b3aa3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:27,545 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:27,545 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=12, startTime=1732619787435; duration=0sec 2024-11-26T11:16:27,545 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:27,545 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:27,545 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:27,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/9934c4c09d3e46beb06787d36e8e8f02, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:27,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/9934c4c09d3e46beb06787d36e8e8f02 is 175, key is test_row_0/A:col10/1732619787437/Put/seqid=0 2024-11-26T11:16:27,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619787511, max timestamp=1732619787545Num uniques 859; 2024-11-26T11:16:27,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619787511, max timestamp=1732619787545Num uniques 859; 2024-11-26T11:16:27,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:27,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:27,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619787511, max timestamp=1732619787545Num uniques 859; 2024-11-26T11:16:27,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:16:27,550 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:27,550 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:27,550 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:27,550 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a3ecae654b0c4e8bb8be21c06ee9686f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/481669eae593493281217398be6f6184, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/73103b0f967248f9b937c18dcd461964, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7146ea6e7b234ed49feeba7fe2f2abf9] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=48.0 K 2024-11-26T11:16:27,551 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a3ecae654b0c4e8bb8be21c06ee9686f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732619782646 2024-11-26T11:16:27,551 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 481669eae593493281217398be6f6184, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732619783823 2024-11-26T11:16:27,551 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 73103b0f967248f9b937c18dcd461964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732619784956 2024-11-26T11:16:27,551 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7146ea6e7b234ed49feeba7fe2f2abf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732619786133 2024-11-26T11:16:27,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742216_1392 (size=31835) 2024-11-26T11:16:27,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742217_1393 (size=31255) 2024-11-26T11:16:27,559 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/7275b61a89db4e2fb94099fec4daa7ff as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7275b61a89db4e2fb94099fec4daa7ff 2024-11-26T11:16:27,562 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:27,563 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/295a53042e40476d9a2827b414e870fc is 50, key is test_row_0/C:col10/1732619786152/Put/seqid=0 2024-11-26T11:16:27,565 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into 7275b61a89db4e2fb94099fec4daa7ff(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:27,565 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:27,565 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=12, startTime=1732619787426; duration=0sec 2024-11-26T11:16:27,565 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:27,565 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742218_1394 (size=12881) 2024-11-26T11:16:27,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732619787582, max timestamp=1732619787617Num uniques 1068; 2024-11-26T11:16:27,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-26T11:16:27,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732619787582, max timestamp=1732619787617Num uniques 1068; 2024-11-26T11:16:27,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732619787582, max timestamp=1732619787617Num uniques 1068; 2024-11-26T11:16:27,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-26T11:16:27,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-26T11:16:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619847677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732619787646, max timestamp=1732619787682Num uniques 1127; 2024-11-26T11:16:27,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-26T11:16:27,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732619787646, max timestamp=1732619787682Num uniques 1127; 2024-11-26T11:16:27,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732619787646, max timestamp=1732619787682Num uniques 1127; 2024-11-26T11:16:27,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-26T11:16:27,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-26T11:16:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619847729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619847741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619847743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619847793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619847853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619847862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619847862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T11:16:27,883 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:27,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-26T11:16:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:27,885 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:27,885 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:27,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:27,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/9934c4c09d3e46beb06787d36e8e8f02 2024-11-26T11:16:27,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/9fec1937b3d44fb49f064ad792c0bb94 is 50, key is test_row_0/B:col10/1732619787437/Put/seqid=0 2024-11-26T11:16:27,975 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/295a53042e40476d9a2827b414e870fc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/295a53042e40476d9a2827b414e870fc 2024-11-26T11:16:27,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742219_1395 (size=12301) 2024-11-26T11:16:27,982 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into 295a53042e40476d9a2827b414e870fc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:27,983 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:27,983 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=12, startTime=1732619787445; duration=0sec 2024-11-26T11:16:27,983 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:27,983 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:28,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619848003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619848073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619848073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619848073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:28,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619848323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619848353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,363 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/9fec1937b3d44fb49f064ad792c0bb94 2024-11-26T11:16:28,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/a53417d8fc4b4251aa9ff7d678f1b083 is 50, key is test_row_0/C:col10/1732619787437/Put/seqid=0 2024-11-26T11:16:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619848393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619848393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619848393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742220_1396 (size=12301) 2024-11-26T11:16:28,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:28,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/a53417d8fc4b4251aa9ff7d678f1b083 2024-11-26T11:16:28,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/9934c4c09d3e46beb06787d36e8e8f02 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02 2024-11-26T11:16:28,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02, entries=150, sequenceid=285, filesize=30.5 K 2024-11-26T11:16:28,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/9fec1937b3d44fb49f064ad792c0bb94 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/9fec1937b3d44fb49f064ad792c0bb94 2024-11-26T11:16:28,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/9fec1937b3d44fb49f064ad792c0bb94, entries=150, sequenceid=285, filesize=12.0 K 2024-11-26T11:16:28,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/a53417d8fc4b4251aa9ff7d678f1b083 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a53417d8fc4b4251aa9ff7d678f1b083 2024-11-26T11:16:28,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a53417d8fc4b4251aa9ff7d678f1b083, entries=150, sequenceid=285, filesize=12.0 K 2024-11-26T11:16:28,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~96.66 KB/98976, currentSize=154.31 KB/158010 for 4772d5b8c16134e9a403e97eb904a2d8 in 1387ms, sequenceid=285, compaction requested=false 2024-11-26T11:16:28,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:28,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=161.02 KB heapSize=449.63 KB 2024-11-26T11:16:28,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732619787728, max timestamp=1732619788849Num uniques 1186; 2024-11-26T11:16:28,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-11-26T11:16:28,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:28,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732619787728, max timestamp=1732619788849Num uniques 1186; 2024-11-26T11:16:28,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:28,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:28,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:28,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:28,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-11-26T11:16:28,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T11:16:28,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f9f708a42f743ba85eda13545360e0f_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:28,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742221_1397 (size=14994) 2024-11-26T11:16:28,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619848932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619848943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619848944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619848948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:28,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:28,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619849053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619849063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619849063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619849063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,267 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619849263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:29,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f9f708a42f743ba85eda13545360e0f_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f9f708a42f743ba85eda13545360e0f_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:29,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6665b640631d44d1867d35398772d2ed, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:29,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6665b640631d44d1867d35398772d2ed is 175, key is test_row_0/A:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:29,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742222_1398 (size=39949) 2024-11-26T11:16:29,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619849283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619849283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619849283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:29,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:29,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619849573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619849603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619849603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619849603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:29,677 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6665b640631d44d1867d35398772d2ed 2024-11-26T11:16:29,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/38238b48e21a445eb33d90d6c0bae422 is 50, key is test_row_0/B:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:29,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:29,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742223_1399 (size=12301) 2024-11-26T11:16:29,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:29,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:29,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:30,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619850093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619850112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:30,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619850123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619850124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:30,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/38238b48e21a445eb33d90d6c0bae422 2024-11-26T11:16:30,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/7f331a0e5d764eafa01e0bdc56b22878 is 50, key is test_row_0/C:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742224_1400 (size=12301) 2024-11-26T11:16:30,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:30,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:30,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:30,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:30,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38534 deadline: 1732619850383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:30,395 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:30,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:30,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/7f331a0e5d764eafa01e0bdc56b22878 2024-11-26T11:16:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6665b640631d44d1867d35398772d2ed as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed 2024-11-26T11:16:30,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed, entries=200, sequenceid=360, filesize=39.0 K 2024-11-26T11:16:30,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/38238b48e21a445eb33d90d6c0bae422 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/38238b48e21a445eb33d90d6c0bae422 2024-11-26T11:16:30,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/38238b48e21a445eb33d90d6c0bae422, entries=150, sequenceid=360, filesize=12.0 K 2024-11-26T11:16:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/7f331a0e5d764eafa01e0bdc56b22878 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7f331a0e5d764eafa01e0bdc56b22878 2024-11-26T11:16:30,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7f331a0e5d764eafa01e0bdc56b22878, entries=150, sequenceid=360, filesize=12.0 K 2024-11-26T11:16:30,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~192.30 KB/196912, currentSize=114.05 KB/116790 for 4772d5b8c16134e9a403e97eb904a2d8 in 1736ms, sequenceid=360, compaction requested=true 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:30,590 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:30,590 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:30,591 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,591 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,591 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a956fac079f345ee90dba15da92b3aa3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/9fec1937b3d44fb49f064ad792c0bb94, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/38238b48e21a445eb33d90d6c0bae422] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=36.6 K 2024-11-26T11:16:30,591 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7275b61a89db4e2fb94099fec4daa7ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=100.6 K 2024-11-26T11:16:30,591 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7275b61a89db4e2fb94099fec4daa7ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed] 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a956fac079f345ee90dba15da92b3aa3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732619786133 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7275b61a89db4e2fb94099fec4daa7ff, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732619786133 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fec1937b3d44fb49f064ad792c0bb94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619787288 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9934c4c09d3e46beb06787d36e8e8f02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619787288 2024-11-26T11:16:30,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 38238b48e21a445eb33d90d6c0bae422, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732619787728 2024-11-26T11:16:30,592 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6665b640631d44d1867d35398772d2ed, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732619787728 2024-11-26T11:16:30,596 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:30,597 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:30,598 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0f992b1b014541a195e07169c672dbb2 is 50, key is test_row_0/B:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:30,600 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112692ab2b1aafd6445c88413833e880e5eb_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:30,601 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112692ab2b1aafd6445c88413833e880e5eb_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:30,601 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112692ab2b1aafd6445c88413833e880e5eb_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:30,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742225_1401 (size=12983) 2024-11-26T11:16:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742226_1402 (size=4469) 2024-11-26T11:16:30,611 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#334 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:30,611 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/71e8d3c8d7a94d52ad2b068978b3fc4b is 175, key is test_row_0/A:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742227_1403 (size=31937) 2024-11-26T11:16:30,626 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/71e8d3c8d7a94d52ad2b068978b3fc4b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/71e8d3c8d7a94d52ad2b068978b3fc4b 2024-11-26T11:16:30,631 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into 71e8d3c8d7a94d52ad2b068978b3fc4b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:30,631 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:30,631 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=13, startTime=1732619790590; duration=0sec 2024-11-26T11:16:30,631 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:30,631 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:30,632 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:30,632 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:30,632 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:30,632 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,632 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/295a53042e40476d9a2827b414e870fc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a53417d8fc4b4251aa9ff7d678f1b083, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7f331a0e5d764eafa01e0bdc56b22878] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=36.6 K 2024-11-26T11:16:30,633 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295a53042e40476d9a2827b414e870fc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732619786133 2024-11-26T11:16:30,633 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a53417d8fc4b4251aa9ff7d678f1b083, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619787288 2024-11-26T11:16:30,633 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f331a0e5d764eafa01e0bdc56b22878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732619787728 2024-11-26T11:16:30,639 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:30,639 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/de44c1f3ade343bca496399f37b6a38a is 50, key is test_row_0/C:col10/1732619787731/Put/seqid=0 2024-11-26T11:16:30,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742228_1404 (size=12983) 2024-11-26T11:16:30,651 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/de44c1f3ade343bca496399f37b6a38a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/de44c1f3ade343bca496399f37b6a38a 2024-11-26T11:16:30,655 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into de44c1f3ade343bca496399f37b6a38a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:30,655 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:30,655 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=13, startTime=1732619790590; duration=0sec 2024-11-26T11:16:30,655 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:30,655 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:30,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T11:16:30,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:30,664 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:30,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112684d15aa501a64c4dac390d050608ae2b_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619788942/Put/seqid=0 2024-11-26T11:16:30,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742229_1405 (size=12454) 2024-11-26T11:16:31,008 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0f992b1b014541a195e07169c672dbb2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0f992b1b014541a195e07169c672dbb2 2024-11-26T11:16:31,012 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into 0f992b1b014541a195e07169c672dbb2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:31,012 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:31,012 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=13, startTime=1732619790590; duration=0sec 2024-11-26T11:16:31,012 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:31,012 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:31,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112684d15aa501a64c4dac390d050608ae2b_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112684d15aa501a64c4dac390d050608ae2b_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:31,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/8f6df0c8627a44e3a1cc87dad71fc588, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:31,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/8f6df0c8627a44e3a1cc87dad71fc588 is 175, key is test_row_0/A:col10/1732619788942/Put/seqid=0 2024-11-26T11:16:31,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742230_1406 (size=31255) 2024-11-26T11:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:31,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. as already flushing 2024-11-26T11:16:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619851142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38518 deadline: 1732619851143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38496 deadline: 1732619851143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,149 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,149 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in 4772d5b8c16134e9a403e97eb904a2d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:16:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619851143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619851253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619851262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619851473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619851473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/8f6df0c8627a44e3a1cc87dad71fc588 2024-11-26T11:16:31,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/a6abf8b4a25b4c4a82de269000ea7d88 is 50, key is test_row_0/B:col10/1732619788942/Put/seqid=0 2024-11-26T11:16:31,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742231_1407 (size=12301) 2024-11-26T11:16:31,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619851782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619851792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:31,899 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/a6abf8b4a25b4c4a82de269000ea7d88 2024-11-26T11:16:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d7409f402d8340d5a9e4d25f08f8ba4b is 50, key is test_row_0/C:col10/1732619788942/Put/seqid=0 2024-11-26T11:16:31,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742232_1408 (size=12301) 2024-11-26T11:16:31,978 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-26T11:16:31,978 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:31,978 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,978 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:31,980 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-26T11:16:31,980 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:31,980 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,981 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:31,981 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-26T11:16:31,981 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:31,981 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,981 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:31,987 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-26T11:16:31,987 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:31,987 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,987 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:31,990 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-26T11:16:31,990 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:31,990 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,991 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:31,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38486 deadline: 1732619852293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:32,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38540 deadline: 1732619852303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:16:32,309 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d7409f402d8340d5a9e4d25f08f8ba4b 2024-11-26T11:16:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/8f6df0c8627a44e3a1cc87dad71fc588 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588 2024-11-26T11:16:32,323 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588, entries=150, sequenceid=382, filesize=30.5 K 2024-11-26T11:16:32,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/a6abf8b4a25b4c4a82de269000ea7d88 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a6abf8b4a25b4c4a82de269000ea7d88 2024-11-26T11:16:32,328 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a6abf8b4a25b4c4a82de269000ea7d88, entries=150, sequenceid=382, filesize=12.0 K 2024-11-26T11:16:32,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d7409f402d8340d5a9e4d25f08f8ba4b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d7409f402d8340d5a9e4d25f08f8ba4b 2024-11-26T11:16:32,332 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d7409f402d8340d5a9e4d25f08f8ba4b, entries=150, sequenceid=382, filesize=12.0 K 2024-11-26T11:16:32,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4772d5b8c16134e9a403e97eb904a2d8 in 1669ms, sequenceid=382, compaction requested=false 2024-11-26T11:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-26T11:16:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-26T11:16:32,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-26T11:16:32,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4490 sec 2024-11-26T11:16:32,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 4.4530 sec 2024-11-26T11:16:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:33,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T11:16:33,168 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-26T11:16:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:33,168 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:33,168 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:33,168 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:33,174 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-26T11:16:33,174 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:33,174 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,174 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:33,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269c2911eeb6db4409bc4557439586d446_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:33,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742233_1409 (size=12454) 2024-11-26T11:16:33,307 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-26T11:16:33,307 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:33,308 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,308 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:33,316 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-26T11:16:33,316 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:33,316 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,316 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:33,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:33,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:33,586 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269c2911eeb6db4409bc4557439586d446_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269c2911eeb6db4409bc4557439586d446_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:33,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/2202db13102448829ac68ed85cb0eeac, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:33,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/2202db13102448829ac68ed85cb0eeac is 175, key is test_row_0/A:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:33,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742234_1410 (size=31255) 2024-11-26T11:16:33,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/2202db13102448829ac68ed85cb0eeac 2024-11-26T11:16:34,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0e1cc21cd1b14ccc961c06f214fc5244 is 50, key is test_row_0/B:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:34,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742235_1411 (size=12301) 2024-11-26T11:16:34,263 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T11:16:34,405 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-26T11:16:34,405 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:16:34,405 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:34,405 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:34,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:34,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0e1cc21cd1b14ccc961c06f214fc5244 2024-11-26T11:16:34,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/095c0db20bf1450d80c0628fce1db667 is 50, key is test_row_0/C:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:34,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742236_1412 (size=12301) 2024-11-26T11:16:34,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/095c0db20bf1450d80c0628fce1db667 2024-11-26T11:16:34,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/2202db13102448829ac68ed85cb0eeac as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac 2024-11-26T11:16:34,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac, entries=150, sequenceid=396, filesize=30.5 K 2024-11-26T11:16:34,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/0e1cc21cd1b14ccc961c06f214fc5244 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0e1cc21cd1b14ccc961c06f214fc5244 2024-11-26T11:16:34,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0e1cc21cd1b14ccc961c06f214fc5244, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T11:16:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/095c0db20bf1450d80c0628fce1db667 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/095c0db20bf1450d80c0628fce1db667 2024-11-26T11:16:34,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/095c0db20bf1450d80c0628fce1db667, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T11:16:34,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 4772d5b8c16134e9a403e97eb904a2d8 in 1670ms, sequenceid=396, compaction requested=true 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:34,837 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:34,837 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4772d5b8c16134e9a403e97eb904a2d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:34,838 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:34,838 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:34,838 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/A is initiating minor compaction (all files) 2024-11-26T11:16:34,838 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/B is initiating minor compaction (all files) 2024-11-26T11:16:34,838 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/A in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:34,838 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/B in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:34,838 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/71e8d3c8d7a94d52ad2b068978b3fc4b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=92.2 K 2024-11-26T11:16:34,838 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0f992b1b014541a195e07169c672dbb2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a6abf8b4a25b4c4a82de269000ea7d88, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0e1cc21cd1b14ccc961c06f214fc5244] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=36.7 K 2024-11-26T11:16:34,838 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:34,838 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/71e8d3c8d7a94d52ad2b068978b3fc4b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac] 2024-11-26T11:16:34,838 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f992b1b014541a195e07169c672dbb2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732619787728 2024-11-26T11:16:34,839 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e8d3c8d7a94d52ad2b068978b3fc4b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732619787728 2024-11-26T11:16:34,839 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a6abf8b4a25b4c4a82de269000ea7d88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732619788924 2024-11-26T11:16:34,839 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f6df0c8627a44e3a1cc87dad71fc588, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732619788924 2024-11-26T11:16:34,839 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1cc21cd1b14ccc961c06f214fc5244, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732619791136 2024-11-26T11:16:34,839 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2202db13102448829ac68ed85cb0eeac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732619791136 2024-11-26T11:16:34,844 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:34,846 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:34,847 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/4c72a12675e3433c83e3d4324abce20d is 50, key is test_row_0/B:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:34,848 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411260234869028e5452ab1a246c41eae9e74_4772d5b8c16134e9a403e97eb904a2d8 store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:34,851 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411260234869028e5452ab1a246c41eae9e74_4772d5b8c16134e9a403e97eb904a2d8, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:34,851 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260234869028e5452ab1a246c41eae9e74_4772d5b8c16134e9a403e97eb904a2d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742237_1413 (size=13085) 2024-11-26T11:16:34,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742238_1414 (size=4469) 2024-11-26T11:16:35,255 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:35,256 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6a693d5303ca4985887269a3855ed87b is 175, key is test_row_0/A:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:35,257 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/4c72a12675e3433c83e3d4324abce20d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4c72a12675e3433c83e3d4324abce20d 2024-11-26T11:16:35,261 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/B of 4772d5b8c16134e9a403e97eb904a2d8 into 4c72a12675e3433c83e3d4324abce20d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:35,261 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:35,261 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/B, priority=13, startTime=1732619794837; duration=0sec 2024-11-26T11:16:35,261 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:35,261 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:B 2024-11-26T11:16:35,261 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:35,262 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:35,262 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 4772d5b8c16134e9a403e97eb904a2d8/C is initiating minor compaction (all files) 2024-11-26T11:16:35,262 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4772d5b8c16134e9a403e97eb904a2d8/C in TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:35,262 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/de44c1f3ade343bca496399f37b6a38a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d7409f402d8340d5a9e4d25f08f8ba4b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/095c0db20bf1450d80c0628fce1db667] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp, totalSize=36.7 K 2024-11-26T11:16:35,262 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting de44c1f3ade343bca496399f37b6a38a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732619787728 2024-11-26T11:16:35,263 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d7409f402d8340d5a9e4d25f08f8ba4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732619788924 2024-11-26T11:16:35,263 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 095c0db20bf1450d80c0628fce1db667, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732619791136 2024-11-26T11:16:35,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742239_1415 (size=32039) 2024-11-26T11:16:35,274 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4772d5b8c16134e9a403e97eb904a2d8#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:35,274 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d54145f08d2947ea80138188b56fb4bd is 50, key is test_row_0/C:col10/1732619791137/Put/seqid=0 2024-11-26T11:16:35,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742240_1416 (size=13085) 2024-11-26T11:16:35,670 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/6a693d5303ca4985887269a3855ed87b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6a693d5303ca4985887269a3855ed87b 2024-11-26T11:16:35,674 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/A of 4772d5b8c16134e9a403e97eb904a2d8 into 6a693d5303ca4985887269a3855ed87b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:35,674 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:35,674 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/A, priority=13, startTime=1732619794837; duration=0sec 2024-11-26T11:16:35,674 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:35,674 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:A 2024-11-26T11:16:35,686 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d54145f08d2947ea80138188b56fb4bd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d54145f08d2947ea80138188b56fb4bd 2024-11-26T11:16:35,690 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4772d5b8c16134e9a403e97eb904a2d8/C of 4772d5b8c16134e9a403e97eb904a2d8 into d54145f08d2947ea80138188b56fb4bd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:35,690 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4772d5b8c16134e9a403e97eb904a2d8: 2024-11-26T11:16:35,690 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., storeName=4772d5b8c16134e9a403e97eb904a2d8/C, priority=13, startTime=1732619794837; duration=0sec 2024-11-26T11:16:35,690 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:35,690 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4772d5b8c16134e9a403e97eb904a2d8:C 2024-11-26T11:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T11:16:36,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1017 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1018 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3045 rows 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1007 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3021 rows 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1014 2024-11-26T11:16:36,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3042 rows 2024-11-26T11:16:36,084 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:16:36,084 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:16:36,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:36,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:36,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:16:36,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T11:16:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T11:16:36,088 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619796088"}]},"ts":"1732619796088"} 2024-11-26T11:16:36,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T11:16:36,089 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T11:16:36,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:16:36,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, UNASSIGN}] 2024-11-26T11:16:36,092 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, UNASSIGN 2024-11-26T11:16:36,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:36,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, UNASSIGN because future has completed 2024-11-26T11:16:36,094 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:16:36,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:16:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T11:16:36,247 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:36,247 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:16:36,247 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing 4772d5b8c16134e9a403e97eb904a2d8, disabling compactions & flushes 2024-11-26T11:16:36,247 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:36,247 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:36,247 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. after waiting 0 ms 2024-11-26T11:16:36,247 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:36,248 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(2902): Flushing 4772d5b8c16134e9a403e97eb904a2d8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T11:16:36,248 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=A 2024-11-26T11:16:36,248 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:36,248 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=B 2024-11-26T11:16:36,248 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:36,248 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4772d5b8c16134e9a403e97eb904a2d8, store=C 2024-11-26T11:16:36,249 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:36,258 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c5bc1e6675a340c5ae4c4780a8b47179_4772d5b8c16134e9a403e97eb904a2d8 is 50, key is test_row_0/A:col10/1732619793304/Put/seqid=0 2024-11-26T11:16:36,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742241_1417 (size=12454) 2024-11-26T11:16:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T11:16:36,663 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:36,667 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c5bc1e6675a340c5ae4c4780a8b47179_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c5bc1e6675a340c5ae4c4780a8b47179_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:36,668 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/29460a442a944df5b6f490d6441e66ef, store: [table=TestAcidGuarantees family=A region=4772d5b8c16134e9a403e97eb904a2d8] 2024-11-26T11:16:36,669 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/29460a442a944df5b6f490d6441e66ef is 175, key is test_row_0/A:col10/1732619793304/Put/seqid=0 2024-11-26T11:16:36,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742242_1418 (size=31255) 2024-11-26T11:16:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T11:16:37,075 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/29460a442a944df5b6f490d6441e66ef 2024-11-26T11:16:37,081 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/755a01367d5046ebae9e83e51dc21ab8 is 50, key is test_row_0/B:col10/1732619793304/Put/seqid=0 2024-11-26T11:16:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742243_1419 (size=12301) 2024-11-26T11:16:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T11:16:37,485 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/755a01367d5046ebae9e83e51dc21ab8 2024-11-26T11:16:37,492 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d9ac388e677f442ca17d3686e80f37ed is 50, key is test_row_0/C:col10/1732619793304/Put/seqid=0 2024-11-26T11:16:37,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742244_1420 (size=12301) 2024-11-26T11:16:37,896 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d9ac388e677f442ca17d3686e80f37ed 2024-11-26T11:16:37,902 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/A/29460a442a944df5b6f490d6441e66ef as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/29460a442a944df5b6f490d6441e66ef 2024-11-26T11:16:37,907 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/29460a442a944df5b6f490d6441e66ef, entries=150, sequenceid=406, filesize=30.5 K 2024-11-26T11:16:37,907 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/B/755a01367d5046ebae9e83e51dc21ab8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/755a01367d5046ebae9e83e51dc21ab8 2024-11-26T11:16:37,912 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/755a01367d5046ebae9e83e51dc21ab8, entries=150, sequenceid=406, filesize=12.0 K 2024-11-26T11:16:37,912 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/.tmp/C/d9ac388e677f442ca17d3686e80f37ed as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d9ac388e677f442ca17d3686e80f37ed 2024-11-26T11:16:37,917 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d9ac388e677f442ca17d3686e80f37ed, entries=150, sequenceid=406, filesize=12.0 K 2024-11-26T11:16:37,917 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4772d5b8c16134e9a403e97eb904a2d8 in 1670ms, sequenceid=406, compaction requested=false 2024-11-26T11:16:37,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ecb640354f614710bcd985788b286f75, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/612732afa30940ab80c2c847f42bc887, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7ef4d38348c9401bab76b7f762809509, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ad049f963d644a288903e0786fa474c6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7275b61a89db4e2fb94099fec4daa7ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/71e8d3c8d7a94d52ad2b068978b3fc4b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac] to archive 2024-11-26T11:16:37,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:16:37,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/44b49e79925547dd90a39d634942661d 2024-11-26T11:16:37,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e20c5fbd940341f4b31ec615e1515e9f 2024-11-26T11:16:37,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4da0d751c10243f0b0818f1ae7405694 2024-11-26T11:16:37,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/4b856bdbcf444416b7b02c33608d3f4f 2024-11-26T11:16:37,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ecb640354f614710bcd985788b286f75 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ecb640354f614710bcd985788b286f75 2024-11-26T11:16:37,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a0e3f000b9b54fc080845e385c112d84 2024-11-26T11:16:37,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/b03497471241478b8976b50d8c13810c 2024-11-26T11:16:37,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/612732afa30940ab80c2c847f42bc887 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/612732afa30940ab80c2c847f42bc887 2024-11-26T11:16:37,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/e771a82984d74c239ef8c4f0e0972e10 2024-11-26T11:16:37,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/5fe049ae100f439c8717dde937601dfb 2024-11-26T11:16:37,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7ef4d38348c9401bab76b7f762809509 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7ef4d38348c9401bab76b7f762809509 2024-11-26T11:16:37,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/858c480b7fc34ba3bd2f5b743366fade 2024-11-26T11:16:37,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/89b1e75bc5ac4f9ca731088ccaff8aa8 2024-11-26T11:16:37,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ad049f963d644a288903e0786fa474c6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/ad049f963d644a288903e0786fa474c6 2024-11-26T11:16:37,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6052be1f405a489a8d4515689e800f82 2024-11-26T11:16:37,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/a5c44b688c184a60998e202533f4f236 2024-11-26T11:16:37,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/c9fbe3208f7a4eda8336cc13deaa7499 2024-11-26T11:16:37,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7275b61a89db4e2fb94099fec4daa7ff to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/7275b61a89db4e2fb94099fec4daa7ff 2024-11-26T11:16:37,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/9934c4c09d3e46beb06787d36e8e8f02 2024-11-26T11:16:37,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6665b640631d44d1867d35398772d2ed 2024-11-26T11:16:37,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/71e8d3c8d7a94d52ad2b068978b3fc4b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/71e8d3c8d7a94d52ad2b068978b3fc4b 2024-11-26T11:16:37,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/8f6df0c8627a44e3a1cc87dad71fc588 2024-11-26T11:16:37,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/2202db13102448829ac68ed85cb0eeac 2024-11-26T11:16:37,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/be162e0fbfce4765a43ad2434f0f5c26, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/64719fd4131042679d834d427f15e225, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e482a949c0ef44d2adc1f6bd05f57113, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e4ff98d323e748f7a3eae8d9e2a1903b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4d0c7dc3eb0342cab1e088613e3eb1e3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/b1cce69c15364375944f7fbb8d7f39a6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/fae3b364b4cd4093b55de496d6cc26a7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/7264ada4ae2648c0bf687222c20c4bc2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/c08ea95439ac4dd9843630ef7d0131e9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/13bd98f556a340d9ad4a39685048e3a1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0bde5dac61454fc886d1f94145c6402b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/88afbfee810442869a648610b7c3cb59, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/86ddc394b67141b8937061e8c4d190ea, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/68f5ab8bd4b74ed09d28b8f8c4922997, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/32e6f56e09a94eaeb493c9b5002f8b02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d203b1e0b18042909072bc02ffd6c3cd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a956fac079f345ee90dba15da92b3aa3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d939773f5e3f4bf28694153f240cb479, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/9fec1937b3d44fb49f064ad792c0bb94, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0f992b1b014541a195e07169c672dbb2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/38238b48e21a445eb33d90d6c0bae422, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a6abf8b4a25b4c4a82de269000ea7d88, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0e1cc21cd1b14ccc961c06f214fc5244] to archive 2024-11-26T11:16:37,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:16:37,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/be162e0fbfce4765a43ad2434f0f5c26 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/be162e0fbfce4765a43ad2434f0f5c26 2024-11-26T11:16:37,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/64719fd4131042679d834d427f15e225 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/64719fd4131042679d834d427f15e225 2024-11-26T11:16:37,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e482a949c0ef44d2adc1f6bd05f57113 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e482a949c0ef44d2adc1f6bd05f57113 2024-11-26T11:16:37,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e4ff98d323e748f7a3eae8d9e2a1903b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/e4ff98d323e748f7a3eae8d9e2a1903b 2024-11-26T11:16:37,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4d0c7dc3eb0342cab1e088613e3eb1e3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4d0c7dc3eb0342cab1e088613e3eb1e3 2024-11-26T11:16:37,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/b1cce69c15364375944f7fbb8d7f39a6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/b1cce69c15364375944f7fbb8d7f39a6 2024-11-26T11:16:37,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/fae3b364b4cd4093b55de496d6cc26a7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/fae3b364b4cd4093b55de496d6cc26a7 2024-11-26T11:16:37,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/7264ada4ae2648c0bf687222c20c4bc2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/7264ada4ae2648c0bf687222c20c4bc2 2024-11-26T11:16:37,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/c08ea95439ac4dd9843630ef7d0131e9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/c08ea95439ac4dd9843630ef7d0131e9 2024-11-26T11:16:37,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/13bd98f556a340d9ad4a39685048e3a1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/13bd98f556a340d9ad4a39685048e3a1 2024-11-26T11:16:37,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0bde5dac61454fc886d1f94145c6402b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0bde5dac61454fc886d1f94145c6402b 2024-11-26T11:16:37,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/88afbfee810442869a648610b7c3cb59 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/88afbfee810442869a648610b7c3cb59 2024-11-26T11:16:37,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/86ddc394b67141b8937061e8c4d190ea to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/86ddc394b67141b8937061e8c4d190ea 2024-11-26T11:16:37,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/68f5ab8bd4b74ed09d28b8f8c4922997 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/68f5ab8bd4b74ed09d28b8f8c4922997 2024-11-26T11:16:37,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/32e6f56e09a94eaeb493c9b5002f8b02 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/32e6f56e09a94eaeb493c9b5002f8b02 2024-11-26T11:16:37,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d203b1e0b18042909072bc02ffd6c3cd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d203b1e0b18042909072bc02ffd6c3cd 2024-11-26T11:16:37,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a956fac079f345ee90dba15da92b3aa3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a956fac079f345ee90dba15da92b3aa3 2024-11-26T11:16:37,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d939773f5e3f4bf28694153f240cb479 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/d939773f5e3f4bf28694153f240cb479 2024-11-26T11:16:37,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/9fec1937b3d44fb49f064ad792c0bb94 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/9fec1937b3d44fb49f064ad792c0bb94 2024-11-26T11:16:37,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0f992b1b014541a195e07169c672dbb2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0f992b1b014541a195e07169c672dbb2 2024-11-26T11:16:37,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/38238b48e21a445eb33d90d6c0bae422 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/38238b48e21a445eb33d90d6c0bae422 2024-11-26T11:16:37,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a6abf8b4a25b4c4a82de269000ea7d88 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/a6abf8b4a25b4c4a82de269000ea7d88 2024-11-26T11:16:37,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0e1cc21cd1b14ccc961c06f214fc5244 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/0e1cc21cd1b14ccc961c06f214fc5244 2024-11-26T11:16:37,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/795cdbae14c74a658180144892d99544, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/412061ed7641495e99636486024cbee4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/1201a8bda7ad4d1f8e015d0e6fb8b559, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/25f474373e8b4aaebbd768a093a4f9f3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/04535f1d1d174d00880e2fa22364105b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/28b0b22c51fd4b5a985cacda7a890a69, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/186b6ff717ad4a98a9ec54e912748831, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/2ea873ce9955498e93eac290443617f7, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/ee5b3016e9e943438b95a6ff45d62da2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/3ef3449edb0a4d2f83b0718105aa2e5b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/f4417afcb9694cf997275ed09f8dee8e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d4ac54dd478648f2bd10a2333f635b87, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a3ecae654b0c4e8bb8be21c06ee9686f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/db4baacf61a1438689d10a3e302119d6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/481669eae593493281217398be6f6184, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/73103b0f967248f9b937c18dcd461964, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/295a53042e40476d9a2827b414e870fc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7146ea6e7b234ed49feeba7fe2f2abf9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a53417d8fc4b4251aa9ff7d678f1b083, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/de44c1f3ade343bca496399f37b6a38a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7f331a0e5d764eafa01e0bdc56b22878, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d7409f402d8340d5a9e4d25f08f8ba4b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/095c0db20bf1450d80c0628fce1db667] to archive 2024-11-26T11:16:37,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:16:37,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/795cdbae14c74a658180144892d99544 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/795cdbae14c74a658180144892d99544 2024-11-26T11:16:37,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/412061ed7641495e99636486024cbee4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/412061ed7641495e99636486024cbee4 2024-11-26T11:16:37,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/1201a8bda7ad4d1f8e015d0e6fb8b559 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/1201a8bda7ad4d1f8e015d0e6fb8b559 2024-11-26T11:16:37,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/25f474373e8b4aaebbd768a093a4f9f3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/25f474373e8b4aaebbd768a093a4f9f3 2024-11-26T11:16:37,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/04535f1d1d174d00880e2fa22364105b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/04535f1d1d174d00880e2fa22364105b 2024-11-26T11:16:37,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/28b0b22c51fd4b5a985cacda7a890a69 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/28b0b22c51fd4b5a985cacda7a890a69 2024-11-26T11:16:37,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/186b6ff717ad4a98a9ec54e912748831 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/186b6ff717ad4a98a9ec54e912748831 2024-11-26T11:16:37,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/2ea873ce9955498e93eac290443617f7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/2ea873ce9955498e93eac290443617f7 2024-11-26T11:16:37,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/ee5b3016e9e943438b95a6ff45d62da2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/ee5b3016e9e943438b95a6ff45d62da2 2024-11-26T11:16:37,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/3ef3449edb0a4d2f83b0718105aa2e5b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/3ef3449edb0a4d2f83b0718105aa2e5b 2024-11-26T11:16:37,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/f4417afcb9694cf997275ed09f8dee8e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/f4417afcb9694cf997275ed09f8dee8e 2024-11-26T11:16:37,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d4ac54dd478648f2bd10a2333f635b87 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d4ac54dd478648f2bd10a2333f635b87 2024-11-26T11:16:37,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a3ecae654b0c4e8bb8be21c06ee9686f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a3ecae654b0c4e8bb8be21c06ee9686f 2024-11-26T11:16:37,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/db4baacf61a1438689d10a3e302119d6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/db4baacf61a1438689d10a3e302119d6 2024-11-26T11:16:37,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/481669eae593493281217398be6f6184 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/481669eae593493281217398be6f6184 2024-11-26T11:16:37,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/73103b0f967248f9b937c18dcd461964 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/73103b0f967248f9b937c18dcd461964 2024-11-26T11:16:37,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/295a53042e40476d9a2827b414e870fc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/295a53042e40476d9a2827b414e870fc 2024-11-26T11:16:37,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7146ea6e7b234ed49feeba7fe2f2abf9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7146ea6e7b234ed49feeba7fe2f2abf9 2024-11-26T11:16:37,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a53417d8fc4b4251aa9ff7d678f1b083 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/a53417d8fc4b4251aa9ff7d678f1b083 2024-11-26T11:16:37,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/de44c1f3ade343bca496399f37b6a38a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/de44c1f3ade343bca496399f37b6a38a 2024-11-26T11:16:37,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7f331a0e5d764eafa01e0bdc56b22878 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/7f331a0e5d764eafa01e0bdc56b22878 2024-11-26T11:16:37,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d7409f402d8340d5a9e4d25f08f8ba4b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d7409f402d8340d5a9e4d25f08f8ba4b 2024-11-26T11:16:37,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/095c0db20bf1450d80c0628fce1db667 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/095c0db20bf1450d80c0628fce1db667 2024-11-26T11:16:37,977 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/recovered.edits/409.seqid, newMaxSeqId=409, maxSeqId=4 2024-11-26T11:16:37,978 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. 2024-11-26T11:16:37,978 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for 4772d5b8c16134e9a403e97eb904a2d8: Waiting for close lock at 1732619796247Running coprocessor pre-close hooks at 1732619796247Disabling compacts and flushes for region at 1732619796247Disabling writes for close at 1732619796247Obtaining lock to block concurrent updates at 1732619796248 (+1 ms)Preparing flush snapshotting stores in 4772d5b8c16134e9a403e97eb904a2d8 at 1732619796248Finished memstore snapshotting TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732619796249 (+1 ms)Flushing stores of TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8. at 1732619796250 (+1 ms)Flushing 4772d5b8c16134e9a403e97eb904a2d8/A: creating writer at 1732619796250Flushing mob file 4772d5b8c16134e9a403e97eb904a2d8/A: appending metadata at 1732619796258 (+8 ms)Flushing mob file 4772d5b8c16134e9a403e97eb904a2d8/A: closing flushed file at 1732619796258Flushing 4772d5b8c16134e9a403e97eb904a2d8/A: appending metadata at 1732619796668 (+410 ms)Flushing 4772d5b8c16134e9a403e97eb904a2d8/A: closing flushed file at 1732619796668Flushing 4772d5b8c16134e9a403e97eb904a2d8/B: creating writer at 1732619797078 (+410 ms)Flushing 4772d5b8c16134e9a403e97eb904a2d8/B: appending metadata at 1732619797081 (+3 ms)Flushing 4772d5b8c16134e9a403e97eb904a2d8/B: closing flushed file at 1732619797081Flushing 4772d5b8c16134e9a403e97eb904a2d8/C: creating writer at 1732619797490 (+409 ms)Flushing 4772d5b8c16134e9a403e97eb904a2d8/C: appending metadata at 1732619797492 (+2 ms)Flushing 4772d5b8c16134e9a403e97eb904a2d8/C: closing flushed file at 1732619797492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d92e43: reopening flushed file at 1732619797901 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2060a0eb: reopening flushed file at 1732619797907 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5be09cdc: reopening flushed file at 1732619797912 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4772d5b8c16134e9a403e97eb904a2d8 in 1670ms, sequenceid=406, compaction requested=false at 1732619797917 (+5 ms)Writing region close event to WAL at 1732619797975 (+58 ms)Running coprocessor post-close hooks at 1732619797978 (+3 ms)Closed at 1732619797978 2024-11-26T11:16:37,980 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed 4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:37,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=4772d5b8c16134e9a403e97eb904a2d8, regionState=CLOSED 2024-11-26T11:16:37,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:16:37,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T11:16:37,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure 4772d5b8c16134e9a403e97eb904a2d8, server=7f3a3edaecc3,37499,1732619676667 in 1.8880 sec 2024-11-26T11:16:37,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-26T11:16:37,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4772d5b8c16134e9a403e97eb904a2d8, UNASSIGN in 1.8920 sec 2024-11-26T11:16:37,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-26T11:16:37,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8950 sec 2024-11-26T11:16:37,988 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619797987"}]},"ts":"1732619797987"} 2024-11-26T11:16:37,989 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T11:16:37,989 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T11:16:37,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9050 sec 2024-11-26T11:16:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T11:16:38,243 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:38,243 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T11:16:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,245 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T11:16:38,245 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,247 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,249 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/recovered.edits] 2024-11-26T11:16:38,252 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/29460a442a944df5b6f490d6441e66ef to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/29460a442a944df5b6f490d6441e66ef 2024-11-26T11:16:38,253 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6a693d5303ca4985887269a3855ed87b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/A/6a693d5303ca4985887269a3855ed87b 2024-11-26T11:16:38,256 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4c72a12675e3433c83e3d4324abce20d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/4c72a12675e3433c83e3d4324abce20d 2024-11-26T11:16:38,257 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/755a01367d5046ebae9e83e51dc21ab8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/B/755a01367d5046ebae9e83e51dc21ab8 2024-11-26T11:16:38,260 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d54145f08d2947ea80138188b56fb4bd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d54145f08d2947ea80138188b56fb4bd 2024-11-26T11:16:38,261 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d9ac388e677f442ca17d3686e80f37ed to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/C/d9ac388e677f442ca17d3686e80f37ed 2024-11-26T11:16:38,262 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/recovered.edits/409.seqid to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8/recovered.edits/409.seqid 2024-11-26T11:16:38,263 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,263 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T11:16:38,263 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T11:16:38,264 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T11:16:38,265 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e09aacc4e4e4e5a854ef20f373e7c3e_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e09aacc4e4e4e5a854ef20f373e7c3e_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,266 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112611ea4d4bc02f49c9bb474c69c6b76849_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112611ea4d4bc02f49c9bb474c69c6b76849_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,267 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261af0b2c015d44056897a2f2ac0d5e863_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261af0b2c015d44056897a2f2ac0d5e863_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f9201f2275e45fe86c0e4960d6902d2_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f9201f2275e45fe86c0e4960d6902d2_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f9f708a42f743ba85eda13545360e0f_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f9f708a42f743ba85eda13545360e0f_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,269 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112684d15aa501a64c4dac390d050608ae2b_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112684d15aa501a64c4dac390d050608ae2b_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,270 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126901ee90e21a3457fb85dc49e390a0668_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126901ee90e21a3457fb85dc49e390a0668_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,270 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269c2911eeb6db4409bc4557439586d446_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269c2911eeb6db4409bc4557439586d446_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,271 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bf1eaee2abf14c089c53cb3aec03cfd8_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bf1eaee2abf14c089c53cb3aec03cfd8_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,271 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c45991ed5a394394a52b69342ac4cc33_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c45991ed5a394394a52b69342ac4cc33_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,272 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c5bc1e6675a340c5ae4c4780a8b47179_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c5bc1e6675a340c5ae4c4780a8b47179_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,273 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cbcd25258f44404da7cca5d2a1131e85_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cbcd25258f44404da7cca5d2a1131e85_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,274 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d4cda46ad2e94d24b24d42b1f612d56f_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d4cda46ad2e94d24b24d42b1f612d56f_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,275 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d5369c2a2811401fa491be3848fb01d9_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d5369c2a2811401fa491be3848fb01d9_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,275 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d85b56690a2a4257af4e6d4ca07f4aff_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d85b56690a2a4257af4e6d4ca07f4aff_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,276 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e30e688488534f2c8aad67626c4424af_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e30e688488534f2c8aad67626c4424af_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e3e8157634d946e59cb89b54a5799851_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e3e8157634d946e59cb89b54a5799851_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fa7ae9ec98d14d6680375ca1a8e94245_4772d5b8c16134e9a403e97eb904a2d8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126fa7ae9ec98d14d6680375ca1a8e94245_4772d5b8c16134e9a403e97eb904a2d8 2024-11-26T11:16:38,278 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T11:16:38,279 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,281 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T11:16:38,283 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T11:16:38,284 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,284 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T11:16:38,284 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732619798284"}]},"ts":"9223372036854775807"} 2024-11-26T11:16:38,285 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T11:16:38,285 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4772d5b8c16134e9a403e97eb904a2d8, NAME => 'TestAcidGuarantees,,1732619768787.4772d5b8c16134e9a403e97eb904a2d8.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T11:16:38,285 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T11:16:38,286 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732619798285"}]},"ts":"9223372036854775807"} 2024-11-26T11:16:38,287 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T11:16:38,287 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-26T11:16:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T11:16:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T11:16:38,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:38,364 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1301850378_22 at /127.0.0.1:36544 [Waiting for operation #634] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1301850378_22 at /127.0.0.1:44972 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_181182078_22 at /127.0.0.1:53496 [Waiting for operation #405] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_181182078_22 at /127.0.0.1:44974 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=470 (was 393) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2678 (was 3212) 2024-11-26T11:16:38,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=470, ProcessCount=11, AvailableMemoryMB=2678 2024-11-26T11:16:38,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:16:38,374 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:16:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T11:16:38,376 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T11:16:38,376 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:38,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-11-26T11:16:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-26T11:16:38,377 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T11:16:38,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742245_1421 (size=1061) 2024-11-26T11:16:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-26T11:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-26T11:16:38,786 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:16:38,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742246_1422 (size=53) 2024-11-26T11:16:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-26T11:16:39,194 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:16:39,194 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 54ea48ed016411b0401034b85aa37157, disabling compactions & flushes 2024-11-26T11:16:39,194 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,194 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,194 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. after waiting 0 ms 2024-11-26T11:16:39,194 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,194 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,194 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54ea48ed016411b0401034b85aa37157: Waiting for close lock at 1732619799194Disabling compacts and flushes for region at 1732619799194Disabling writes for close at 1732619799194Writing region close event to WAL at 1732619799194Closed at 1732619799194 2024-11-26T11:16:39,258 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T11:16:39,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732619799258"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732619799258"}]},"ts":"1732619799258"} 2024-11-26T11:16:39,261 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T11:16:39,262 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T11:16:39,262 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619799262"}]},"ts":"1732619799262"} 2024-11-26T11:16:39,264 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T11:16:39,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, ASSIGN}] 2024-11-26T11:16:39,267 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, ASSIGN 2024-11-26T11:16:39,269 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, ASSIGN; state=OFFLINE, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=false 2024-11-26T11:16:39,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=54ea48ed016411b0401034b85aa37157, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:39,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, ASSIGN because future has completed 2024-11-26T11:16:39,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:16:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-26T11:16:39,580 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,580 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:16:39,581 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,582 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:16:39,582 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,582 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,583 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,585 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:39,585 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ea48ed016411b0401034b85aa37157 columnFamilyName A 2024-11-26T11:16:39,585 DEBUG [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:39,586 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.HStore(327): Store=54ea48ed016411b0401034b85aa37157/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:39,586 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,587 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:39,587 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ea48ed016411b0401034b85aa37157 columnFamilyName B 2024-11-26T11:16:39,587 DEBUG [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:39,588 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.HStore(327): Store=54ea48ed016411b0401034b85aa37157/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:39,588 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,589 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:16:39,589 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ea48ed016411b0401034b85aa37157 columnFamilyName C 2024-11-26T11:16:39,590 DEBUG [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:16:39,590 INFO [StoreOpener-54ea48ed016411b0401034b85aa37157-1 {}] regionserver.HStore(327): Store=54ea48ed016411b0401034b85aa37157/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:16:39,590 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,590 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,591 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,592 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,593 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,593 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,593 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:16:39,595 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,597 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:16:39,597 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 54ea48ed016411b0401034b85aa37157; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62778781, jitterRate=-0.06452326476573944}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:16:39,597 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:39,598 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 54ea48ed016411b0401034b85aa37157: Running coprocessor pre-open hook at 1732619799582Writing region info on filesystem at 1732619799582Initializing all the Stores at 1732619799583 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619799583Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619799583Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619799583Cleaning up temporary data from old regions at 1732619799593 (+10 ms)Running coprocessor post-open hooks at 1732619799597 (+4 ms)Region opened successfully at 1732619799598 (+1 ms) 2024-11-26T11:16:39,599 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., pid=106, masterSystemTime=1732619799577 2024-11-26T11:16:39,601 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,601 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:39,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=54ea48ed016411b0401034b85aa37157, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:39,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:16:39,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T11:16:39,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure 54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 in 181 msec 2024-11-26T11:16:39,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-26T11:16:39,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, ASSIGN in 342 msec 2024-11-26T11:16:39,610 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T11:16:39,611 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619799610"}]},"ts":"1732619799610"} 2024-11-26T11:16:39,613 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T11:16:39,614 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T11:16:39,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2400 sec 2024-11-26T11:16:40,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-26T11:16:40,534 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:40,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,540 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c91513d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640d89d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:16:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:16:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b6d66e2 2024-11-26T11:16:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:16:40,549 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:16:40,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,552 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3348c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567266b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b6f1bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae406aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,560 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5824132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe4fd4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,563 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec2328c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,565 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a61b28a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,568 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18310024, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48b40a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,572 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18924c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,573 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76aede6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,576 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c887ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,578 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a26097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5969a2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6adbe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33044441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11f24dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:16:40,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:16:40,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:16:40,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:16:40,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:16:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66907725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:16:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:16:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:16:40,591 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:16:40,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdf3b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:16:40,592 DEBUG [Thread-1904 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,592 DEBUG [Thread-1906 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,592 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,597 DEBUG [Thread-1913 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,598 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,598 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,600 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,600 DEBUG [Thread-1915 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,601 DEBUG [Thread-1902 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,601 DEBUG [Thread-1908 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,601 DEBUG [Thread-1908 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,601 DEBUG [Thread-1919 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,601 DEBUG [Thread-1919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,601 DEBUG [Thread-1917 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:16:40,602 DEBUG [Thread-1917 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,603 DEBUG [Thread-1917 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,604 DEBUG [Thread-1910 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,604 DEBUG [Thread-1910 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:40,604 DEBUG [Thread-1915 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:16:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,604 DEBUG [Thread-1915 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:16:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:16:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-26T11:16:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,607 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2] 2024-11-26T11:16:40,608 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:40,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:16:40,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:40,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:40,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/d6e950503a0b4231ab4ce47a02bcc74e is 50, key is test_row_0/A:col10/1732619800605/Put/seqid=0 2024-11-26T11:16:40,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732619800620, max timestamp=1732619800638Num uniques 859; 2024-11-26T11:16:40,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732619800620, max timestamp=1732619800638Num uniques 859; 2024-11-26T11:16:40,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T11:16:40,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T11:16:40,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732619800620, max timestamp=1732619800638Num uniques 859; 2024-11-26T11:16:40,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T11:16:40,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742247_1423 (size=16681) 2024-11-26T11:16:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619860657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619860664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619860665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619860666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619860672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:40,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:40,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:40,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:40,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:40,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:40,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:40,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619860783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619860783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619860783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619860783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619860783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:40,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:40,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619860993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619860993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619860993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619860993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619860993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/d6e950503a0b4231ab4ce47a02bcc74e 2024-11-26T11:16:41,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/288d780fcf5c4a87956aa63c75d5bd49 is 50, key is test_row_0/B:col10/1732619800605/Put/seqid=0 2024-11-26T11:16:41,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742248_1424 (size=12001) 2024-11-26T11:16:41,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/288d780fcf5c4a87956aa63c75d5bd49 2024-11-26T11:16:41,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/30a79fc08fe443eaa05e0baf29a7334b is 50, key is test_row_0/C:col10/1732619800605/Put/seqid=0 2024-11-26T11:16:41,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742249_1425 (size=12001) 2024-11-26T11:16:41,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:41,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619861303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619861303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619861303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619861303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619861303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/30a79fc08fe443eaa05e0baf29a7334b 2024-11-26T11:16:41,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/d6e950503a0b4231ab4ce47a02bcc74e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d6e950503a0b4231ab4ce47a02bcc74e 2024-11-26T11:16:41,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d6e950503a0b4231ab4ce47a02bcc74e, entries=250, sequenceid=11, filesize=16.3 K 2024-11-26T11:16:41,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/288d780fcf5c4a87956aa63c75d5bd49 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/288d780fcf5c4a87956aa63c75d5bd49 2024-11-26T11:16:41,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/288d780fcf5c4a87956aa63c75d5bd49, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T11:16:41,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/30a79fc08fe443eaa05e0baf29a7334b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/30a79fc08fe443eaa05e0baf29a7334b 2024-11-26T11:16:41,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/30a79fc08fe443eaa05e0baf29a7334b, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T11:16:41,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:41,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 54ea48ed016411b0401034b85aa37157 in 918ms, sequenceid=11, compaction requested=false 2024-11-26T11:16:41,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:41,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:41,678 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/31f6dff9eed44da8b0ed4bdecbaf01f8 is 50, key is test_row_0/A:col10/1732619800663/Put/seqid=0 2024-11-26T11:16:41,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742250_1426 (size=12001) 2024-11-26T11:16:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:41,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619861833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619861832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619861836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619861834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619861845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619861952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619861952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619861953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619861953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619861953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,089 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/31f6dff9eed44da8b0ed4bdecbaf01f8 2024-11-26T11:16:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/bbd0b17279af4239a6cb716042d664b2 is 50, key is test_row_0/B:col10/1732619800663/Put/seqid=0 2024-11-26T11:16:42,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742251_1427 (size=12001) 2024-11-26T11:16:42,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619862173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619862173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619862173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619862173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619862173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,502 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/bbd0b17279af4239a6cb716042d664b2 2024-11-26T11:16:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619862493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619862493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619862493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619862493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619862493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7afc51b52c704712ae1720ea6839a0ff is 50, key is test_row_0/C:col10/1732619800663/Put/seqid=0 2024-11-26T11:16:42,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742252_1428 (size=12001) 2024-11-26T11:16:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:42,912 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7afc51b52c704712ae1720ea6839a0ff 2024-11-26T11:16:42,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/31f6dff9eed44da8b0ed4bdecbaf01f8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/31f6dff9eed44da8b0ed4bdecbaf01f8 2024-11-26T11:16:42,920 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/31f6dff9eed44da8b0ed4bdecbaf01f8, entries=150, sequenceid=45, filesize=11.7 K 2024-11-26T11:16:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/bbd0b17279af4239a6cb716042d664b2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/bbd0b17279af4239a6cb716042d664b2 2024-11-26T11:16:42,924 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/bbd0b17279af4239a6cb716042d664b2, entries=150, sequenceid=45, filesize=11.7 K 2024-11-26T11:16:42,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7afc51b52c704712ae1720ea6839a0ff as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7afc51b52c704712ae1720ea6839a0ff 2024-11-26T11:16:42,928 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7afc51b52c704712ae1720ea6839a0ff, entries=150, sequenceid=45, filesize=11.7 K 2024-11-26T11:16:42,928 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 54ea48ed016411b0401034b85aa37157 in 1250ms, sequenceid=45, compaction requested=false 2024-11-26T11:16:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-26T11:16:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-26T11:16:42,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-26T11:16:42,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3220 sec 2024-11-26T11:16:42,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3280 sec 2024-11-26T11:16:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:43,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T11:16:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:43,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:43,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/e957acdc71374eb6b45ac928ee636384 is 50, key is test_row_0/A:col10/1732619803018/Put/seqid=0 2024-11-26T11:16:43,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742253_1429 (size=16681) 2024-11-26T11:16:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619863051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619863052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619863054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619863055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619863056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619863173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619863173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619863173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619863173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619863173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619863383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619863383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619863383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619863393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/e957acdc71374eb6b45ac928ee636384 2024-11-26T11:16:43,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/df5b16dfd9d84e68877ff2362f738b67 is 50, key is test_row_0/B:col10/1732619803018/Put/seqid=0 2024-11-26T11:16:43,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742254_1430 (size=12001) 2024-11-26T11:16:43,502 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:16:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619863693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619863693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619863702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619863703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619863703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:43,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/df5b16dfd9d84e68877ff2362f738b67 2024-11-26T11:16:43,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/0dde918b8c0a4b6c9c5b625a58ed1b7b is 50, key is test_row_0/C:col10/1732619803018/Put/seqid=0 2024-11-26T11:16:43,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742255_1431 (size=12001) 2024-11-26T11:16:44,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619864202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:44,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619864202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619864213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619864213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619864213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:44,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/0dde918b8c0a4b6c9c5b625a58ed1b7b 2024-11-26T11:16:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/e957acdc71374eb6b45ac928ee636384 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e957acdc71374eb6b45ac928ee636384 2024-11-26T11:16:44,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e957acdc71374eb6b45ac928ee636384, entries=250, sequenceid=57, filesize=16.3 K 2024-11-26T11:16:44,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/df5b16dfd9d84e68877ff2362f738b67 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/df5b16dfd9d84e68877ff2362f738b67 2024-11-26T11:16:44,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/df5b16dfd9d84e68877ff2362f738b67, entries=150, sequenceid=57, filesize=11.7 K 2024-11-26T11:16:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/0dde918b8c0a4b6c9c5b625a58ed1b7b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0dde918b8c0a4b6c9c5b625a58ed1b7b 2024-11-26T11:16:44,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0dde918b8c0a4b6c9c5b625a58ed1b7b, entries=150, sequenceid=57, filesize=11.7 K 2024-11-26T11:16:44,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 54ea48ed016411b0401034b85aa37157 in 1269ms, sequenceid=57, compaction requested=true 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:44,287 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:44,287 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:44,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:44,288 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:44,288 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:16:44,288 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:44,288 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/288d780fcf5c4a87956aa63c75d5bd49, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/bbd0b17279af4239a6cb716042d664b2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/df5b16dfd9d84e68877ff2362f738b67] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.2 K 2024-11-26T11:16:44,288 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 288d780fcf5c4a87956aa63c75d5bd49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732619800601 2024-11-26T11:16:44,289 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd0b17279af4239a6cb716042d664b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732619800663 2024-11-26T11:16:44,289 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting df5b16dfd9d84e68877ff2362f738b67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619801844 2024-11-26T11:16:44,290 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:44,291 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:16:44,291 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:44,291 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d6e950503a0b4231ab4ce47a02bcc74e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/31f6dff9eed44da8b0ed4bdecbaf01f8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e957acdc71374eb6b45ac928ee636384] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=44.3 K 2024-11-26T11:16:44,291 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6e950503a0b4231ab4ce47a02bcc74e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732619800599 2024-11-26T11:16:44,291 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31f6dff9eed44da8b0ed4bdecbaf01f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732619800663 2024-11-26T11:16:44,291 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e957acdc71374eb6b45ac928ee636384, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619801830 2024-11-26T11:16:44,296 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#357 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:44,296 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/e569db345c564f698996ce6e1a4ec11a is 50, key is test_row_0/B:col10/1732619803018/Put/seqid=0 2024-11-26T11:16:44,298 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:44,298 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/2959c18bdea74dcf90a3233c41dafdee is 50, key is test_row_0/A:col10/1732619803018/Put/seqid=0 2024-11-26T11:16:44,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742256_1432 (size=12104) 2024-11-26T11:16:44,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742257_1433 (size=12104) 2024-11-26T11:16:44,310 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/e569db345c564f698996ce6e1a4ec11a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/e569db345c564f698996ce6e1a4ec11a 2024-11-26T11:16:44,314 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into e569db345c564f698996ce6e1a4ec11a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:44,314 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:44,314 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=13, startTime=1732619804287; duration=0sec 2024-11-26T11:16:44,314 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:44,314 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:16:44,314 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:44,314 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:44,315 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:16:44,315 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:44,315 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/30a79fc08fe443eaa05e0baf29a7334b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7afc51b52c704712ae1720ea6839a0ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0dde918b8c0a4b6c9c5b625a58ed1b7b] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.2 K 2024-11-26T11:16:44,315 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a79fc08fe443eaa05e0baf29a7334b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732619800601 2024-11-26T11:16:44,315 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 7afc51b52c704712ae1720ea6839a0ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732619800663 2024-11-26T11:16:44,315 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dde918b8c0a4b6c9c5b625a58ed1b7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619801844 2024-11-26T11:16:44,320 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:44,321 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7484c3334bd24a28a0ab00975d58f814 is 50, key is test_row_0/C:col10/1732619803018/Put/seqid=0 2024-11-26T11:16:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742258_1434 (size=12104) 2024-11-26T11:16:44,706 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/2959c18bdea74dcf90a3233c41dafdee as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2959c18bdea74dcf90a3233c41dafdee 2024-11-26T11:16:44,709 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into 2959c18bdea74dcf90a3233c41dafdee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:44,709 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:44,709 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=13, startTime=1732619804287; duration=0sec 2024-11-26T11:16:44,709 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:44,709 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:16:44,727 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7484c3334bd24a28a0ab00975d58f814 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7484c3334bd24a28a0ab00975d58f814 2024-11-26T11:16:44,731 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into 7484c3334bd24a28a0ab00975d58f814(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:44,731 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:44,731 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=13, startTime=1732619804287; duration=0sec 2024-11-26T11:16:44,731 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:44,731 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:16:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T11:16:44,773 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:44,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-26T11:16:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T11:16:44,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:44,776 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:44,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T11:16:44,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T11:16:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:44,928 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:16:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:44,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/b7303d7b4da34aa4ad1b92444f608202 is 50, key is test_row_0/A:col10/1732619803055/Put/seqid=0 2024-11-26T11:16:44,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742259_1435 (size=12001) 2024-11-26T11:16:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T11:16:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:45,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619865241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619865242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619865243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619865244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619865245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,349 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/b7303d7b4da34aa4ad1b92444f608202 2024-11-26T11:16:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d4a7ac680bbb4c0aaa662911a8ca8db3 is 50, key is test_row_0/B:col10/1732619803055/Put/seqid=0 2024-11-26T11:16:45,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742260_1436 (size=12001) 2024-11-26T11:16:45,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619865353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619865353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619865362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619865362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619865362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T11:16:45,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619865563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619865563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619865572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619865572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619865583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,760 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d4a7ac680bbb4c0aaa662911a8ca8db3 2024-11-26T11:16:45,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4eb022c16ffc42b5820f2144280aa4c9 is 50, key is test_row_0/C:col10/1732619803055/Put/seqid=0 2024-11-26T11:16:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742261_1437 (size=12001) 2024-11-26T11:16:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619865873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619865873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619865892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619865892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619865892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:45,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T11:16:46,169 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4eb022c16ffc42b5820f2144280aa4c9 2024-11-26T11:16:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/b7303d7b4da34aa4ad1b92444f608202 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b7303d7b4da34aa4ad1b92444f608202 2024-11-26T11:16:46,177 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b7303d7b4da34aa4ad1b92444f608202, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T11:16:46,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d4a7ac680bbb4c0aaa662911a8ca8db3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4a7ac680bbb4c0aaa662911a8ca8db3 2024-11-26T11:16:46,181 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4a7ac680bbb4c0aaa662911a8ca8db3, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T11:16:46,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4eb022c16ffc42b5820f2144280aa4c9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4eb022c16ffc42b5820f2144280aa4c9 2024-11-26T11:16:46,184 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4eb022c16ffc42b5820f2144280aa4c9, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T11:16:46,185 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54ea48ed016411b0401034b85aa37157 in 1257ms, sequenceid=80, compaction requested=false 2024-11-26T11:16:46,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:46,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:46,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-26T11:16:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-26T11:16:46,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-26T11:16:46,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-26T11:16:46,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4140 sec 2024-11-26T11:16:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:46,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:46,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/0284ef1bf29143f19d78d0b1ff2b9752 is 50, key is test_row_0/A:col10/1732619806384/Put/seqid=0 2024-11-26T11:16:46,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742262_1438 (size=14341) 2024-11-26T11:16:46,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619866431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619866431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619866432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619866433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619866434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619866542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619866553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619866553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619866553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619866553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619866763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619866763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619866763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619866763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619866763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:46,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/0284ef1bf29143f19d78d0b1ff2b9752 2024-11-26T11:16:46,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/9268fefd39f741b3b3691078e98a5877 is 50, key is test_row_0/B:col10/1732619806384/Put/seqid=0 2024-11-26T11:16:46,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742263_1439 (size=12001) 2024-11-26T11:16:46,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/9268fefd39f741b3b3691078e98a5877 2024-11-26T11:16:46,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/449dd0be86f24448bca047c7187697b4 is 50, key is test_row_0/C:col10/1732619806384/Put/seqid=0 2024-11-26T11:16:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742264_1440 (size=12001) 2024-11-26T11:16:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T11:16:46,923 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:46,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-26T11:16:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T11:16:46,925 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:46,926 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:46,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T11:16:47,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T11:16:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619867072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619867072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619867072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619867073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619867083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/449dd0be86f24448bca047c7187697b4 2024-11-26T11:16:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/0284ef1bf29143f19d78d0b1ff2b9752 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/0284ef1bf29143f19d78d0b1ff2b9752 2024-11-26T11:16:47,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/0284ef1bf29143f19d78d0b1ff2b9752, entries=200, sequenceid=93, filesize=14.0 K 2024-11-26T11:16:47,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T11:16:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:47,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/9268fefd39f741b3b3691078e98a5877 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9268fefd39f741b3b3691078e98a5877 2024-11-26T11:16:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:47,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9268fefd39f741b3b3691078e98a5877, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T11:16:47,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/449dd0be86f24448bca047c7187697b4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/449dd0be86f24448bca047c7187697b4 2024-11-26T11:16:47,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/449dd0be86f24448bca047c7187697b4, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T11:16:47,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54ea48ed016411b0401034b85aa37157 in 853ms, sequenceid=93, compaction requested=true 2024-11-26T11:16:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:47,239 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:47,239 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:47,240 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:47,240 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:16:47,240 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,240 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:47,240 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:16:47,240 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2959c18bdea74dcf90a3233c41dafdee, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b7303d7b4da34aa4ad1b92444f608202, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/0284ef1bf29143f19d78d0b1ff2b9752] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=37.5 K 2024-11-26T11:16:47,240 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,240 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/e569db345c564f698996ce6e1a4ec11a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4a7ac680bbb4c0aaa662911a8ca8db3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9268fefd39f741b3b3691078e98a5877] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.3 K 2024-11-26T11:16:47,241 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 2959c18bdea74dcf90a3233c41dafdee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619801844 2024-11-26T11:16:47,241 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e569db345c564f698996ce6e1a4ec11a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619801844 2024-11-26T11:16:47,241 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b7303d7b4da34aa4ad1b92444f608202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732619803049 2024-11-26T11:16:47,241 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a7ac680bbb4c0aaa662911a8ca8db3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732619803049 2024-11-26T11:16:47,241 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0284ef1bf29143f19d78d0b1ff2b9752, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732619805242 2024-11-26T11:16:47,241 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9268fefd39f741b3b3691078e98a5877, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732619805242 2024-11-26T11:16:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T11:16:47,248 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:47,249 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6eab571acb09467981ef5983319c481f is 50, key is test_row_0/B:col10/1732619806384/Put/seqid=0 2024-11-26T11:16:47,249 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:47,250 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/be90618f41344d05af2d76b695dccfcd is 50, key is test_row_0/A:col10/1732619806384/Put/seqid=0 2024-11-26T11:16:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742265_1441 (size=12207) 2024-11-26T11:16:47,258 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6eab571acb09467981ef5983319c481f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6eab571acb09467981ef5983319c481f 2024-11-26T11:16:47,262 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into 6eab571acb09467981ef5983319c481f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:47,262 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:47,262 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=13, startTime=1732619807239; duration=0sec 2024-11-26T11:16:47,262 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:47,262 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:16:47,263 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:47,264 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:47,264 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:16:47,264 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,264 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7484c3334bd24a28a0ab00975d58f814, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4eb022c16ffc42b5820f2144280aa4c9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/449dd0be86f24448bca047c7187697b4] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.3 K 2024-11-26T11:16:47,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7484c3334bd24a28a0ab00975d58f814, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732619801844 2024-11-26T11:16:47,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb022c16ffc42b5820f2144280aa4c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732619803049 2024-11-26T11:16:47,265 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449dd0be86f24448bca047c7187697b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732619805242 2024-11-26T11:16:47,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742266_1442 (size=12207) 2024-11-26T11:16:47,272 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:47,273 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b8c06dc354d240c0bf08984c8e878120 is 50, key is test_row_0/C:col10/1732619806384/Put/seqid=0 2024-11-26T11:16:47,276 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/be90618f41344d05af2d76b695dccfcd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/be90618f41344d05af2d76b695dccfcd 2024-11-26T11:16:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742267_1443 (size=12207) 2024-11-26T11:16:47,293 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into be90618f41344d05af2d76b695dccfcd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:47,293 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:47,293 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=13, startTime=1732619807239; duration=0sec 2024-11-26T11:16:47,293 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:47,293 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:16:47,305 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b8c06dc354d240c0bf08984c8e878120 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b8c06dc354d240c0bf08984c8e878120 2024-11-26T11:16:47,309 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into b8c06dc354d240c0bf08984c8e878120(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:47,309 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:47,309 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=13, startTime=1732619807240; duration=0sec 2024-11-26T11:16:47,310 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:47,310 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:16:47,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T11:16:47,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:47,384 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T11:16:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:47,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/6f20293e8d76407ca106dcf248c3aa25 is 50, key is test_row_0/A:col10/1732619806433/Put/seqid=0 2024-11-26T11:16:47,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742268_1444 (size=12001) 2024-11-26T11:16:47,409 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/6f20293e8d76407ca106dcf248c3aa25 2024-11-26T11:16:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6e591eb49b3749c7865ad67e519ed362 is 50, key is test_row_0/B:col10/1732619806433/Put/seqid=0 2024-11-26T11:16:47,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742269_1445 (size=12001) 2024-11-26T11:16:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T11:16:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:47,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:47,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619867603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619867607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619867608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619867609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619867616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619867722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619867723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619867723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619867723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619867723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6e591eb49b3749c7865ad67e519ed362 2024-11-26T11:16:47,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/24e38755ee274652b2319fa47a3d5c23 is 50, key is test_row_0/C:col10/1732619806433/Put/seqid=0 2024-11-26T11:16:47,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742270_1446 (size=12001) 2024-11-26T11:16:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619867933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619867933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619867933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619867933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619867933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T11:16:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619868242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619868243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619868243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619868243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619868252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,261 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/24e38755ee274652b2319fa47a3d5c23 2024-11-26T11:16:48,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/6f20293e8d76407ca106dcf248c3aa25 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6f20293e8d76407ca106dcf248c3aa25 2024-11-26T11:16:48,273 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6f20293e8d76407ca106dcf248c3aa25, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T11:16:48,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6e591eb49b3749c7865ad67e519ed362 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6e591eb49b3749c7865ad67e519ed362 2024-11-26T11:16:48,278 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6e591eb49b3749c7865ad67e519ed362, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T11:16:48,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/24e38755ee274652b2319fa47a3d5c23 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/24e38755ee274652b2319fa47a3d5c23 2024-11-26T11:16:48,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/24e38755ee274652b2319fa47a3d5c23, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T11:16:48,286 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 54ea48ed016411b0401034b85aa37157 in 903ms, sequenceid=115, compaction requested=false 2024-11-26T11:16:48,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:48,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:48,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-26T11:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-26T11:16:48,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-26T11:16:48,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3610 sec 2024-11-26T11:16:48,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3660 sec 2024-11-26T11:16:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:48,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T11:16:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:48,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/fd1ddb3b4e5846b6a951451587b4ad2d is 50, key is test_row_0/A:col10/1732619808765/Put/seqid=0 2024-11-26T11:16:48,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742271_1447 (size=14391) 2024-11-26T11:16:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619868784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619868791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619868792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619868795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619868796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619868902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619868903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619868913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619868913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619868913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T11:16:49,083 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:49,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-26T11:16:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T11:16:49,084 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:49,085 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:49,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619869113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619869113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619869133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619869133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619869133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/fd1ddb3b4e5846b6a951451587b4ad2d 2024-11-26T11:16:49,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/cc86308650334564a3d3e360cdedbe56 is 50, key is test_row_0/B:col10/1732619808765/Put/seqid=0 2024-11-26T11:16:49,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742272_1448 (size=12051) 2024-11-26T11:16:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T11:16:49,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:49,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:49,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:49,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T11:16:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619869422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619869422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619869453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619869453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619869453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/cc86308650334564a3d3e360cdedbe56 2024-11-26T11:16:49,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/a74f085b89414736bf284112d69369e9 is 50, key is test_row_0/C:col10/1732619808765/Put/seqid=0 2024-11-26T11:16:49,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742273_1449 (size=12051) 2024-11-26T11:16:49,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:49,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T11:16:49,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:49,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:49,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:49,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619869933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619869943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619869962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619869972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619869973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:49,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/a74f085b89414736bf284112d69369e9 2024-11-26T11:16:50,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/fd1ddb3b4e5846b6a951451587b4ad2d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/fd1ddb3b4e5846b6a951451587b4ad2d 2024-11-26T11:16:50,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/fd1ddb3b4e5846b6a951451587b4ad2d, entries=200, sequenceid=129, filesize=14.1 K 2024-11-26T11:16:50,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/cc86308650334564a3d3e360cdedbe56 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/cc86308650334564a3d3e360cdedbe56 2024-11-26T11:16:50,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/cc86308650334564a3d3e360cdedbe56, entries=150, sequenceid=129, filesize=11.8 K 2024-11-26T11:16:50,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/a74f085b89414736bf284112d69369e9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/a74f085b89414736bf284112d69369e9 2024-11-26T11:16:50,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/a74f085b89414736bf284112d69369e9, entries=150, sequenceid=129, filesize=11.8 K 2024-11-26T11:16:50,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 54ea48ed016411b0401034b85aa37157 in 1251ms, sequenceid=129, compaction requested=true 2024-11-26T11:16:50,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,017 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:50,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:50,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:50,018 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:50,018 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:16:50,018 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,018 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/be90618f41344d05af2d76b695dccfcd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6f20293e8d76407ca106dcf248c3aa25, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/fd1ddb3b4e5846b6a951451587b4ad2d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=37.7 K 2024-11-26T11:16:50,018 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:50,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:50,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:50,019 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting be90618f41344d05af2d76b695dccfcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732619805242 2024-11-26T11:16:50,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:50,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:50,019 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f20293e8d76407ca106dcf248c3aa25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732619806418 2024-11-26T11:16:50,019 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1ddb3b4e5846b6a951451587b4ad2d, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732619807606 2024-11-26T11:16:50,019 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:50,019 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:16:50,019 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,019 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6eab571acb09467981ef5983319c481f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6e591eb49b3749c7865ad67e519ed362, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/cc86308650334564a3d3e360cdedbe56] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.4 K 2024-11-26T11:16:50,020 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eab571acb09467981ef5983319c481f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732619805242 2024-11-26T11:16:50,020 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e591eb49b3749c7865ad67e519ed362, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732619806418 2024-11-26T11:16:50,020 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting cc86308650334564a3d3e360cdedbe56, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732619807607 2024-11-26T11:16:50,028 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:50,029 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/ee5f9b1f49524f00904ba275db757d91 is 50, key is test_row_0/B:col10/1732619808765/Put/seqid=0 2024-11-26T11:16:50,038 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:50,039 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/529e54aac29844f9946b168a8965b066 is 50, key is test_row_0/A:col10/1732619808765/Put/seqid=0 2024-11-26T11:16:50,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742274_1450 (size=12359) 2024-11-26T11:16:50,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742275_1451 (size=12359) 2024-11-26T11:16:50,057 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/529e54aac29844f9946b168a8965b066 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/529e54aac29844f9946b168a8965b066 2024-11-26T11:16:50,061 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into 529e54aac29844f9946b168a8965b066(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:50,061 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,061 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=13, startTime=1732619810017; duration=0sec 2024-11-26T11:16:50,062 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:50,062 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:16:50,062 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:50,062 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:50,062 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:16:50,062 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,063 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b8c06dc354d240c0bf08984c8e878120, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/24e38755ee274652b2319fa47a3d5c23, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/a74f085b89414736bf284112d69369e9] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.4 K 2024-11-26T11:16:50,063 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c06dc354d240c0bf08984c8e878120, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732619805242 2024-11-26T11:16:50,063 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e38755ee274652b2319fa47a3d5c23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732619806418 2024-11-26T11:16:50,064 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74f085b89414736bf284112d69369e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732619807607 2024-11-26T11:16:50,069 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#377 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:50,069 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/fade5d5c372f412c80d3140105cdae64 is 50, key is test_row_0/C:col10/1732619808765/Put/seqid=0 2024-11-26T11:16:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742276_1452 (size=12359) 2024-11-26T11:16:50,075 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/fade5d5c372f412c80d3140105cdae64 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/fade5d5c372f412c80d3140105cdae64 2024-11-26T11:16:50,078 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into fade5d5c372f412c80d3140105cdae64(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:50,079 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,079 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=13, startTime=1732619810019; duration=0sec 2024-11-26T11:16:50,079 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:50,079 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:16:50,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:50,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/42513eb1bf464db7893ac0262ceaca5b is 50, key is test_row_0/A:col10/1732619808794/Put/seqid=0 2024-11-26T11:16:50,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742277_1453 (size=12151) 2024-11-26T11:16:50,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/42513eb1bf464db7893ac0262ceaca5b 2024-11-26T11:16:50,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d41528cc44af497394a90cfd79997445 is 50, key is test_row_0/B:col10/1732619808794/Put/seqid=0 2024-11-26T11:16:50,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742278_1454 (size=12151) 2024-11-26T11:16:50,175 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d41528cc44af497394a90cfd79997445 2024-11-26T11:16:50,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/0cc82d105acc40e99e9a7fc29075661d is 50, key is test_row_0/C:col10/1732619808794/Put/seqid=0 2024-11-26T11:16:50,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742279_1455 (size=12151) 2024-11-26T11:16:50,187 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/0cc82d105acc40e99e9a7fc29075661d 2024-11-26T11:16:50,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/42513eb1bf464db7893ac0262ceaca5b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/42513eb1bf464db7893ac0262ceaca5b 2024-11-26T11:16:50,193 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/42513eb1bf464db7893ac0262ceaca5b, entries=150, sequenceid=149, filesize=11.9 K 2024-11-26T11:16:50,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d41528cc44af497394a90cfd79997445 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d41528cc44af497394a90cfd79997445 2024-11-26T11:16:50,197 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d41528cc44af497394a90cfd79997445, entries=150, sequenceid=149, filesize=11.9 K 2024-11-26T11:16:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/0cc82d105acc40e99e9a7fc29075661d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0cc82d105acc40e99e9a7fc29075661d 2024-11-26T11:16:50,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0cc82d105acc40e99e9a7fc29075661d, entries=150, sequenceid=149, filesize=11.9 K 2024-11-26T11:16:50,201 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 54ea48ed016411b0401034b85aa37157 in 45ms, sequenceid=149, compaction requested=false 2024-11-26T11:16:50,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-26T11:16:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-26T11:16:50,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-26T11:16:50,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1170 sec 2024-11-26T11:16:50,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1220 sec 2024-11-26T11:16:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T11:16:50,223 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:50,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:50,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-26T11:16:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T11:16:50,225 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:50,225 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:50,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T11:16:50,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T11:16:50,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-26T11:16:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-26T11:16:50,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T11:16:50,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 153 msec 2024-11-26T11:16:50,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 158 msec 2024-11-26T11:16:50,447 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/ee5f9b1f49524f00904ba275db757d91 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/ee5f9b1f49524f00904ba275db757d91 2024-11-26T11:16:50,451 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into ee5f9b1f49524f00904ba275db757d91(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:50,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,451 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=13, startTime=1732619810018; duration=0sec 2024-11-26T11:16:50,451 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:50,452 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:16:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T11:16:50,543 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:50,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-26T11:16:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T11:16:50,546 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:50,547 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:50,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T11:16:50,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T11:16:50,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-26T11:16:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-26T11:16:50,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-26T11:16:50,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-26T11:16:50,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 161 msec 2024-11-26T11:16:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T11:16:50,863 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:50,863 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-26T11:16:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:50,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:50,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:50,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:50,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:16:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:50,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/f061ea892be44b88889f1dbcb94af29e is 50, key is test_row_0/A:col10/1732619810949/Put/seqid=0 2024-11-26T11:16:50,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742280_1456 (size=14541) 2024-11-26T11:16:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:51,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619811007, max timestamp=1732619811012Num uniques 859; 2024-11-26T11:16:51,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619811007, max timestamp=1732619811012Num uniques 859; 2024-11-26T11:16:51,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:51,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:51,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619811007, max timestamp=1732619811012Num uniques 859; 2024-11-26T11:16:51,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:51,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619871039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619871040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619871041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619871050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619871053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619871163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619871163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619871163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619871163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619871172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:51,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/f061ea892be44b88889f1dbcb94af29e 2024-11-26T11:16:51,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6753596f93454e2792571c74352f5811 is 50, key is test_row_0/B:col10/1732619810949/Put/seqid=0 2024-11-26T11:16:51,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742281_1457 (size=12151) 2024-11-26T11:16:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619871373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619871373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619871373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619871373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619871393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:51,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619871693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619871693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619871693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619871693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619871713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:51,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6753596f93454e2792571c74352f5811 2024-11-26T11:16:51,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b6b2761aab404237acec6b38b2b063d4 is 50, key is test_row_0/C:col10/1732619810949/Put/seqid=0 2024-11-26T11:16:51,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742282_1458 (size=12151) 2024-11-26T11:16:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:51,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:52,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b6b2761aab404237acec6b38b2b063d4 2024-11-26T11:16:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/f061ea892be44b88889f1dbcb94af29e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f061ea892be44b88889f1dbcb94af29e 2024-11-26T11:16:52,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f061ea892be44b88889f1dbcb94af29e, entries=200, sequenceid=162, filesize=14.2 K 2024-11-26T11:16:52,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/6753596f93454e2792571c74352f5811 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6753596f93454e2792571c74352f5811 2024-11-26T11:16:52,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6753596f93454e2792571c74352f5811, entries=150, sequenceid=162, filesize=11.9 K 2024-11-26T11:16:52,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b6b2761aab404237acec6b38b2b063d4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b6b2761aab404237acec6b38b2b063d4 2024-11-26T11:16:52,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b6b2761aab404237acec6b38b2b063d4, entries=150, sequenceid=162, filesize=11.9 K 2024-11-26T11:16:52,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 54ea48ed016411b0401034b85aa37157 in 1237ms, sequenceid=162, compaction requested=true 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:52,195 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:52,195 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:52,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:52,196 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:52,196 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:16:52,196 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,196 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/529e54aac29844f9946b168a8965b066, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/42513eb1bf464db7893ac0262ceaca5b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f061ea892be44b88889f1dbcb94af29e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=38.1 K 2024-11-26T11:16:52,196 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:52,196 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:16:52,196 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,197 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/ee5f9b1f49524f00904ba275db757d91, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d41528cc44af497394a90cfd79997445, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6753596f93454e2792571c74352f5811] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.8 K 2024-11-26T11:16:52,197 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529e54aac29844f9946b168a8965b066, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732619807607 2024-11-26T11:16:52,197 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5f9b1f49524f00904ba275db757d91, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732619807607 2024-11-26T11:16:52,197 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42513eb1bf464db7893ac0262ceaca5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732619808790 2024-11-26T11:16:52,197 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d41528cc44af497394a90cfd79997445, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732619808790 2024-11-26T11:16:52,197 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f061ea892be44b88889f1dbcb94af29e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619810946 2024-11-26T11:16:52,197 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6753596f93454e2792571c74352f5811, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619810949 2024-11-26T11:16:52,203 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:52,203 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/110e1ad002194d2887662113a7b044ac is 50, key is test_row_0/A:col10/1732619810949/Put/seqid=0 2024-11-26T11:16:52,205 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:52,205 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/379858e8162042858a0498d3aae69032 is 50, key is test_row_0/B:col10/1732619810949/Put/seqid=0 2024-11-26T11:16:52,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742284_1460 (size=12561) 2024-11-26T11:16:52,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742283_1459 (size=12561) 2024-11-26T11:16:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:52,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732619811039, max timestamp=1732619812219Num uniques 1018; 2024-11-26T11:16:52,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T11:16:52,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732619811039, max timestamp=1732619812219Num uniques 1018; 2024-11-26T11:16:52,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T11:16:52,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-26T11:16:52,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732619811039, max timestamp=1732619812219Num uniques 1018; 2024-11-26T11:16:52,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T11:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:52,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:52,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/d1e38bfa8d164e80898483fa6165f52c is 50, key is test_row_0/A:col10/1732619812221/Put/seqid=0 2024-11-26T11:16:52,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742285_1461 (size=19321) 2024-11-26T11:16:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619812266, max timestamp=1732619812283Num uniques 859; 2024-11-26T11:16:52,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619812266, max timestamp=1732619812283Num uniques 859; 2024-11-26T11:16:52,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:52,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:52,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619812266, max timestamp=1732619812283Num uniques 859; 2024-11-26T11:16:52,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:52,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619812334, max timestamp=1732619812355Num uniques 1018; 2024-11-26T11:16:52,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:16:52,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619812334, max timestamp=1732619812355Num uniques 1018; 2024-11-26T11:16:52,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:16:52,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619812334, max timestamp=1732619812355Num uniques 1018; 2024-11-26T11:16:52,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:16:52,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619872394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619872395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619872396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619872405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732619812373, max timestamp=1732619812412Num uniques 1077; 2024-11-26T11:16:52,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-26T11:16:52,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732619812373, max timestamp=1732619812412Num uniques 1077; 2024-11-26T11:16:52,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732619812373, max timestamp=1732619812412Num uniques 1077; 2024-11-26T11:16:52,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-26T11:16:52,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-26T11:16:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619872530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619872531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619872532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619872533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619872534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,621 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/379858e8162042858a0498d3aae69032 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/379858e8162042858a0498d3aae69032 2024-11-26T11:16:52,621 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/110e1ad002194d2887662113a7b044ac as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/110e1ad002194d2887662113a7b044ac 2024-11-26T11:16:52,624 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into 379858e8162042858a0498d3aae69032(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:52,624 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into 110e1ad002194d2887662113a7b044ac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:52,624 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:52,624 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:52,624 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=13, startTime=1732619812195; duration=0sec 2024-11-26T11:16:52,624 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=13, startTime=1732619812195; duration=0sec 2024-11-26T11:16:52,624 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:52,625 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:52,625 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:16:52,625 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:16:52,625 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:52,626 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:52,626 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:16:52,626 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,626 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/fade5d5c372f412c80d3140105cdae64, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0cc82d105acc40e99e9a7fc29075661d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b6b2761aab404237acec6b38b2b063d4] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=35.8 K 2024-11-26T11:16:52,626 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting fade5d5c372f412c80d3140105cdae64, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732619807607 2024-11-26T11:16:52,626 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc82d105acc40e99e9a7fc29075661d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732619808790 2024-11-26T11:16:52,627 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b2761aab404237acec6b38b2b063d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619810949 2024-11-26T11:16:52,631 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:52,631 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/1a17553e3e834883b192d678dfe27b06 is 50, key is test_row_0/C:col10/1732619810949/Put/seqid=0 2024-11-26T11:16:52,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742286_1462 (size=12561) 2024-11-26T11:16:52,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/d1e38bfa8d164e80898483fa6165f52c 2024-11-26T11:16:52,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/586dc568c700405e99533854560da5a5 is 50, key is test_row_0/B:col10/1732619812221/Put/seqid=0 2024-11-26T11:16:52,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742287_1463 (size=12151) 2024-11-26T11:16:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619872652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619872653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619872653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619872653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619872653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:52,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619872873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619872873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619872873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619872873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619872873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:53,042 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/1a17553e3e834883b192d678dfe27b06 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1a17553e3e834883b192d678dfe27b06 2024-11-26T11:16:53,045 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into 1a17553e3e834883b192d678dfe27b06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:53,045 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:53,045 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=13, startTime=1732619812195; duration=0sec 2024-11-26T11:16:53,045 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:53,045 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:16:53,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/586dc568c700405e99533854560da5a5 2024-11-26T11:16:53,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/1936d82972414cea97041195628c963d is 50, key is test_row_0/C:col10/1732619812221/Put/seqid=0 2024-11-26T11:16:53,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742288_1464 (size=12151) 2024-11-26T11:16:53,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:53,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:53,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619873183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619873183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619873183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619873183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619873183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:53,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:53,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/1936d82972414cea97041195628c963d 2024-11-26T11:16:53,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:53,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/d1e38bfa8d164e80898483fa6165f52c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d1e38bfa8d164e80898483fa6165f52c 2024-11-26T11:16:53,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d1e38bfa8d164e80898483fa6165f52c, entries=300, sequenceid=202, filesize=18.9 K 2024-11-26T11:16:53,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/586dc568c700405e99533854560da5a5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/586dc568c700405e99533854560da5a5 2024-11-26T11:16:53,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/586dc568c700405e99533854560da5a5, entries=150, sequenceid=202, filesize=11.9 K 2024-11-26T11:16:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/1936d82972414cea97041195628c963d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1936d82972414cea97041195628c963d 2024-11-26T11:16:53,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1936d82972414cea97041195628c963d, entries=150, sequenceid=202, filesize=11.9 K 2024-11-26T11:16:53,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for 54ea48ed016411b0401034b85aa37157 in 1252ms, sequenceid=202, compaction requested=false 2024-11-26T11:16:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:53,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T11:16:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:53,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=147.60 KB heapSize=409.08 KB 2024-11-26T11:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:53,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/b1d7f145c51a422bb6d374a2a15b7dc4 is 50, key is test_row_0/A:col10/1732619812533/Put/seqid=0 2024-11-26T11:16:53,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742289_1465 (size=12301) 2024-11-26T11:16:53,628 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/b1d7f145c51a422bb6d374a2a15b7dc4 2024-11-26T11:16:53,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/32c4147e3b08449f85b6b75ad00b8daf is 50, key is test_row_0/B:col10/1732619812533/Put/seqid=0 2024-11-26T11:16:53,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742290_1466 (size=12301) 2024-11-26T11:16:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:53,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619873706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619873707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619873708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619873709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619873712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619873823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619873823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619873823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619873824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:53,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619873832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/32c4147e3b08449f85b6b75ad00b8daf 2024-11-26T11:16:54,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4477ad839a8e4f8b8056809b4255c64c is 50, key is test_row_0/C:col10/1732619812533/Put/seqid=0 2024-11-26T11:16:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742291_1467 (size=12301) 2024-11-26T11:16:54,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619874042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619874042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619874043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619874043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619874043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619874353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619874362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619874362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619874362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619874363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,448 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4477ad839a8e4f8b8056809b4255c64c 2024-11-26T11:16:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/b1d7f145c51a422bb6d374a2a15b7dc4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b1d7f145c51a422bb6d374a2a15b7dc4 2024-11-26T11:16:54,455 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b1d7f145c51a422bb6d374a2a15b7dc4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T11:16:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/32c4147e3b08449f85b6b75ad00b8daf as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/32c4147e3b08449f85b6b75ad00b8daf 2024-11-26T11:16:54,459 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/32c4147e3b08449f85b6b75ad00b8daf, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T11:16:54,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4477ad839a8e4f8b8056809b4255c64c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4477ad839a8e4f8b8056809b4255c64c 2024-11-26T11:16:54,462 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4477ad839a8e4f8b8056809b4255c64c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T11:16:54,463 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~409.03 KB/418848, currentSize=40.25 KB/41220 for 54ea48ed016411b0401034b85aa37157 in 847ms, sequenceid=275, compaction requested=true 2024-11-26T11:16:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-26T11:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-26T11:16:54,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-26T11:16:54,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5980 sec 2024-11-26T11:16:54,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 3.6020 sec 2024-11-26T11:16:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:54,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:16:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:54,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:54,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:54,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:54,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:54,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:54,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/48c2c48bd9314d8089da506577a0185f is 50, key is test_row_0/A:col10/1732619813711/Put/seqid=0 2024-11-26T11:16:54,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742292_1468 (size=12297) 2024-11-26T11:16:54,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/48c2c48bd9314d8089da506577a0185f 2024-11-26T11:16:54,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/9e29e3bfe07046f8ab662195ee7f55fd is 50, key is test_row_0/B:col10/1732619813711/Put/seqid=0 2024-11-26T11:16:54,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742293_1469 (size=9857) 2024-11-26T11:16:54,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619814901, max timestamp=1732619814918Num uniques 859; 2024-11-26T11:16:54,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619814901, max timestamp=1732619814918Num uniques 859; 2024-11-26T11:16:54,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:16:54,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:16:54,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732619814901, max timestamp=1732619814918Num uniques 859; 2024-11-26T11:16:54,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T11:16:54,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619874947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619874953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619874955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619874956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619874957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T11:16:55,023 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:55,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-26T11:16:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T11:16:55,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:55,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:55,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:55,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619875062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619875073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619875073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619875073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619875073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T11:16:55,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T11:16:55,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:55,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619875273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/9e29e3bfe07046f8ab662195ee7f55fd 2024-11-26T11:16:55,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/3c7a4dfb5edb41bc955ace612149ad0a is 50, key is test_row_0/C:col10/1732619813711/Put/seqid=0 2024-11-26T11:16:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619875293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619875293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619875293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619875293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742294_1470 (size=9857) 2024-11-26T11:16:55,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T11:16:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T11:16:55,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T11:16:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619875593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619875612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619875612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619875612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619875613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:55,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T11:16:55,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:55,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T11:16:55,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/3c7a4dfb5edb41bc955ace612149ad0a 2024-11-26T11:16:55,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/48c2c48bd9314d8089da506577a0185f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/48c2c48bd9314d8089da506577a0185f 2024-11-26T11:16:55,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/48c2c48bd9314d8089da506577a0185f, entries=150, sequenceid=285, filesize=12.0 K 2024-11-26T11:16:55,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/9e29e3bfe07046f8ab662195ee7f55fd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9e29e3bfe07046f8ab662195ee7f55fd 2024-11-26T11:16:55,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9e29e3bfe07046f8ab662195ee7f55fd, entries=100, sequenceid=285, filesize=9.6 K 2024-11-26T11:16:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/3c7a4dfb5edb41bc955ace612149ad0a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/3c7a4dfb5edb41bc955ace612149ad0a 2024-11-26T11:16:55,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/3c7a4dfb5edb41bc955ace612149ad0a, entries=100, sequenceid=285, filesize=9.6 K 2024-11-26T11:16:55,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 54ea48ed016411b0401034b85aa37157 in 849ms, sequenceid=285, compaction requested=true 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:55,726 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:55,726 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:55,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:55,727 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:55,727 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:55,727 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:16:55,727 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:16:55,727 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,727 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,727 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/379858e8162042858a0498d3aae69032, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/586dc568c700405e99533854560da5a5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/32c4147e3b08449f85b6b75ad00b8daf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9e29e3bfe07046f8ab662195ee7f55fd] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=45.8 K 2024-11-26T11:16:55,727 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/110e1ad002194d2887662113a7b044ac, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d1e38bfa8d164e80898483fa6165f52c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b1d7f145c51a422bb6d374a2a15b7dc4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/48c2c48bd9314d8089da506577a0185f] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=55.2 K 2024-11-26T11:16:55,728 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 379858e8162042858a0498d3aae69032, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619810949 2024-11-26T11:16:55,728 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 110e1ad002194d2887662113a7b044ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619810949 2024-11-26T11:16:55,729 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e38bfa8d164e80898483fa6165f52c, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732619811039 2024-11-26T11:16:55,729 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 586dc568c700405e99533854560da5a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732619811051 2024-11-26T11:16:55,729 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d7f145c51a422bb6d374a2a15b7dc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732619812445 2024-11-26T11:16:55,729 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c4147e3b08449f85b6b75ad00b8daf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732619812445 2024-11-26T11:16:55,729 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c2c48bd9314d8089da506577a0185f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619813708 2024-11-26T11:16:55,729 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e29e3bfe07046f8ab662195ee7f55fd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619813711 2024-11-26T11:16:55,737 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:55,738 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/169be7530fd44ee482cf2230c82065b0 is 50, key is test_row_0/B:col10/1732619813711/Put/seqid=0 2024-11-26T11:16:55,747 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:55,748 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/563f8c92f6e044298dbbc42aac407d84 is 50, key is test_row_0/A:col10/1732619813711/Put/seqid=0 2024-11-26T11:16:55,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742295_1471 (size=12847) 2024-11-26T11:16:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742296_1472 (size=12847) 2024-11-26T11:16:55,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:55,790 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:55,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/12f451180e704ccc9b2a4f5cd3bb1f54 is 50, key is test_row_0/A:col10/1732619814952/Put/seqid=0 2024-11-26T11:16:55,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742297_1473 (size=12301) 2024-11-26T11:16:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619876133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619876133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619876134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619876138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619876139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,161 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/169be7530fd44ee482cf2230c82065b0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/169be7530fd44ee482cf2230c82065b0 2024-11-26T11:16:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T11:16:56,164 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into 169be7530fd44ee482cf2230c82065b0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:56,164 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:56,164 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=12, startTime=1732619815726; duration=0sec 2024-11-26T11:16:56,164 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:56,164 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:16:56,165 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:16:56,165 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:16:56,165 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:16:56,166 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:56,166 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1a17553e3e834883b192d678dfe27b06, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1936d82972414cea97041195628c963d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4477ad839a8e4f8b8056809b4255c64c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/3c7a4dfb5edb41bc955ace612149ad0a] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=45.8 K 2024-11-26T11:16:56,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a17553e3e834883b192d678dfe27b06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732619810949 2024-11-26T11:16:56,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1936d82972414cea97041195628c963d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732619811051 2024-11-26T11:16:56,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4477ad839a8e4f8b8056809b4255c64c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732619812445 2024-11-26T11:16:56,167 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c7a4dfb5edb41bc955ace612149ad0a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619813711 2024-11-26T11:16:56,173 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:56,174 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/d5ea4f826c45489aa3df8415d29e11dd is 50, key is test_row_0/C:col10/1732619813711/Put/seqid=0 2024-11-26T11:16:56,177 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/563f8c92f6e044298dbbc42aac407d84 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/563f8c92f6e044298dbbc42aac407d84 2024-11-26T11:16:56,180 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into 563f8c92f6e044298dbbc42aac407d84(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:56,180 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:56,180 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=12, startTime=1732619815726; duration=0sec 2024-11-26T11:16:56,180 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:56,180 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:16:56,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742298_1474 (size=12847) 2024-11-26T11:16:56,198 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/12f451180e704ccc9b2a4f5cd3bb1f54 2024-11-26T11:16:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d4ff51c9568243e49f278ee8f8e199c5 is 50, key is test_row_0/B:col10/1732619814952/Put/seqid=0 2024-11-26T11:16:56,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742299_1475 (size=12301) 2024-11-26T11:16:56,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619876243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619876252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619876252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619876263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619876462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619876462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619876462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619876473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,588 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/d5ea4f826c45489aa3df8415d29e11dd as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/d5ea4f826c45489aa3df8415d29e11dd 2024-11-26T11:16:56,591 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into d5ea4f826c45489aa3df8415d29e11dd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:56,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:56,591 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=12, startTime=1732619815726; duration=0sec 2024-11-26T11:16:56,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:56,591 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:16:56,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d4ff51c9568243e49f278ee8f8e199c5 2024-11-26T11:16:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/62c4d32178be4ebe833918e6d5ddf3f8 is 50, key is test_row_0/C:col10/1732619814952/Put/seqid=0 2024-11-26T11:16:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742300_1476 (size=12301) 2024-11-26T11:16:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619876783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619876783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619876783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619876792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,026 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/62c4d32178be4ebe833918e6d5ddf3f8 2024-11-26T11:16:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/12f451180e704ccc9b2a4f5cd3bb1f54 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/12f451180e704ccc9b2a4f5cd3bb1f54 2024-11-26T11:16:57,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/12f451180e704ccc9b2a4f5cd3bb1f54, entries=150, sequenceid=320, filesize=12.0 K 2024-11-26T11:16:57,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/d4ff51c9568243e49f278ee8f8e199c5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4ff51c9568243e49f278ee8f8e199c5 2024-11-26T11:16:57,036 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4ff51c9568243e49f278ee8f8e199c5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-26T11:16:57,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/62c4d32178be4ebe833918e6d5ddf3f8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/62c4d32178be4ebe833918e6d5ddf3f8 2024-11-26T11:16:57,039 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/62c4d32178be4ebe833918e6d5ddf3f8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-26T11:16:57,040 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 54ea48ed016411b0401034b85aa37157 in 1250ms, sequenceid=320, compaction requested=false 2024-11-26T11:16:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-26T11:16:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-26T11:16:57,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-26T11:16:57,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0150 sec 2024-11-26T11:16:57,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.0190 sec 2024-11-26T11:16:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:57,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T11:16:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:57,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/22332123240b4ddaa7d2781852cd1cf8 is 50, key is test_row_0/A:col10/1732619816121/Put/seqid=0 2024-11-26T11:16:57,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742301_1477 (size=14741) 2024-11-26T11:16:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T11:16:57,173 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:57,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-26T11:16:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T11:16:57,174 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:57,175 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:57,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T11:16:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619817257, max timestamp=1732619817290Num uniques 859; 2024-11-26T11:16:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619817257, max timestamp=1732619817290Num uniques 859; 2024-11-26T11:16:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619817257, max timestamp=1732619817290Num uniques 859; 2024-11-26T11:16:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:16:57,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T11:16:57,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:57,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619877327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619877328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619877330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619877343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619877344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619877453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619877463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T11:16:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T11:16:57,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/22332123240b4ddaa7d2781852cd1cf8 2024-11-26T11:16:57,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/5e408d09e43e42019f7702383239e47d is 50, key is test_row_0/B:col10/1732619816121/Put/seqid=0 2024-11-26T11:16:57,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742302_1478 (size=12301) 2024-11-26T11:16:57,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/5e408d09e43e42019f7702383239e47d 2024-11-26T11:16:57,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/42dc8f210f154bee998c9dd30307847c is 50, key is test_row_0/C:col10/1732619816121/Put/seqid=0 2024-11-26T11:16:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742303_1479 (size=12301) 2024-11-26T11:16:57,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T11:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619877662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619877673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619877673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619877673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619877673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T11:16:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T11:16:57,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T11:16:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:57,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:57,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/42dc8f210f154bee998c9dd30307847c 2024-11-26T11:16:57,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619877982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/22332123240b4ddaa7d2781852cd1cf8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/22332123240b4ddaa7d2781852cd1cf8 2024-11-26T11:16:58,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/22332123240b4ddaa7d2781852cd1cf8, entries=200, sequenceid=334, filesize=14.4 K 2024-11-26T11:16:58,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/5e408d09e43e42019f7702383239e47d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/5e408d09e43e42019f7702383239e47d 2024-11-26T11:16:58,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/5e408d09e43e42019f7702383239e47d, entries=150, sequenceid=334, filesize=12.0 K 2024-11-26T11:16:58,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/42dc8f210f154bee998c9dd30307847c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/42dc8f210f154bee998c9dd30307847c 2024-11-26T11:16:58,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619877993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619877993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619877993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619877993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/42dc8f210f154bee998c9dd30307847c, entries=150, sequenceid=334, filesize=12.0 K 2024-11-26T11:16:58,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 54ea48ed016411b0401034b85aa37157 in 853ms, sequenceid=334, compaction requested=true 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:58,011 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:58,011 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:16:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:58,012 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:58,012 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:58,012 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:16:58,012 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:16:58,012 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:58,012 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:58,012 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/563f8c92f6e044298dbbc42aac407d84, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/12f451180e704ccc9b2a4f5cd3bb1f54, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/22332123240b4ddaa7d2781852cd1cf8] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=39.0 K 2024-11-26T11:16:58,012 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/169be7530fd44ee482cf2230c82065b0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4ff51c9568243e49f278ee8f8e199c5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/5e408d09e43e42019f7702383239e47d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=36.6 K 2024-11-26T11:16:58,012 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563f8c92f6e044298dbbc42aac407d84, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619812522 2024-11-26T11:16:58,012 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 169be7530fd44ee482cf2230c82065b0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619812522 2024-11-26T11:16:58,013 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ff51c9568243e49f278ee8f8e199c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732619814952 2024-11-26T11:16:58,013 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f451180e704ccc9b2a4f5cd3bb1f54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732619814952 2024-11-26T11:16:58,013 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e408d09e43e42019f7702383239e47d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732619816121 2024-11-26T11:16:58,013 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22332123240b4ddaa7d2781852cd1cf8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732619816121 2024-11-26T11:16:58,018 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:58,019 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/a8fa5763109047c0b2836e85b84205f8 is 50, key is test_row_0/B:col10/1732619816121/Put/seqid=0 2024-11-26T11:16:58,021 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:58,022 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/34b6525f20f7440aafe1f4a45cbb9a4a is 50, key is test_row_0/A:col10/1732619816121/Put/seqid=0 2024-11-26T11:16:58,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742304_1480 (size=12949) 2024-11-26T11:16:58,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742305_1481 (size=12949) 2024-11-26T11:16:58,034 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/a8fa5763109047c0b2836e85b84205f8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/a8fa5763109047c0b2836e85b84205f8 2024-11-26T11:16:58,038 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into a8fa5763109047c0b2836e85b84205f8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:58,038 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:58,038 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=13, startTime=1732619818011; duration=0sec 2024-11-26T11:16:58,038 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:16:58,038 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:16:58,038 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:16:58,039 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:16:58,039 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:16:58,039 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:58,039 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/d5ea4f826c45489aa3df8415d29e11dd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/62c4d32178be4ebe833918e6d5ddf3f8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/42dc8f210f154bee998c9dd30307847c] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=36.6 K 2024-11-26T11:16:58,040 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/34b6525f20f7440aafe1f4a45cbb9a4a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/34b6525f20f7440aafe1f4a45cbb9a4a 2024-11-26T11:16:58,040 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ea4f826c45489aa3df8415d29e11dd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732619812522 2024-11-26T11:16:58,040 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c4d32178be4ebe833918e6d5ddf3f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732619814952 2024-11-26T11:16:58,040 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 42dc8f210f154bee998c9dd30307847c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732619816121 2024-11-26T11:16:58,046 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into 34b6525f20f7440aafe1f4a45cbb9a4a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:58,047 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:58,047 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=13, startTime=1732619818011; duration=0sec 2024-11-26T11:16:58,047 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:58,047 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:16:58,051 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:16:58,051 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/372fdcb3675b428fa10c8246d05da2fc is 50, key is test_row_0/C:col10/1732619816121/Put/seqid=0 2024-11-26T11:16:58,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742306_1482 (size=12949) 2024-11-26T11:16:58,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T11:16:58,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:58,089 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-26T11:16:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:16:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/f025d1f78e484954b35c28f09b77fe95 is 50, key is test_row_0/A:col10/1732619817343/Put/seqid=0 2024-11-26T11:16:58,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742307_1483 (size=12301) 2024-11-26T11:16:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T11:16:58,461 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/372fdcb3675b428fa10c8246d05da2fc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/372fdcb3675b428fa10c8246d05da2fc 2024-11-26T11:16:58,465 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into 372fdcb3675b428fa10c8246d05da2fc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:16:58,465 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:58,465 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=13, startTime=1732619818011; duration=0sec 2024-11-26T11:16:58,465 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:16:58,465 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:16:58,496 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/f025d1f78e484954b35c28f09b77fe95 2024-11-26T11:16:58,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/4b864aa24ef8434abb4476a11d9bc01a is 50, key is test_row_0/B:col10/1732619817343/Put/seqid=0 2024-11-26T11:16:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:58,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:58,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742308_1484 (size=12301) 2024-11-26T11:16:58,513 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/4b864aa24ef8434abb4476a11d9bc01a 2024-11-26T11:16:58,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4fa13b24dddf412e8c33c5e6463023eb is 50, key is test_row_0/C:col10/1732619817343/Put/seqid=0 2024-11-26T11:16:58,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742309_1485 (size=12301) 2024-11-26T11:16:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619878523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619878524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619878525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619878526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619878540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619878642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619878642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619878642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619878653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619878653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619878863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619878863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619878863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619878873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619878873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:58,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:58,930 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4fa13b24dddf412e8c33c5e6463023eb 2024-11-26T11:16:58,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/f025d1f78e484954b35c28f09b77fe95 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f025d1f78e484954b35c28f09b77fe95 2024-11-26T11:16:58,936 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f025d1f78e484954b35c28f09b77fe95, entries=150, sequenceid=372, filesize=12.0 K 2024-11-26T11:16:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/4b864aa24ef8434abb4476a11d9bc01a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/4b864aa24ef8434abb4476a11d9bc01a 2024-11-26T11:16:58,941 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/4b864aa24ef8434abb4476a11d9bc01a, entries=150, sequenceid=372, filesize=12.0 K 2024-11-26T11:16:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/4fa13b24dddf412e8c33c5e6463023eb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4fa13b24dddf412e8c33c5e6463023eb 2024-11-26T11:16:58,947 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4fa13b24dddf412e8c33c5e6463023eb, entries=150, sequenceid=372, filesize=12.0 K 2024-11-26T11:16:58,948 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 54ea48ed016411b0401034b85aa37157 in 858ms, sequenceid=372, compaction requested=false 2024-11-26T11:16:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:16:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-26T11:16:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-26T11:16:58,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-26T11:16:58,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-26T11:16:58,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7780 sec 2024-11-26T11:16:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:16:59,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T11:16:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:16:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:16:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:16:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:16:59,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/307f8415f6ea41b9ad4e8e1c7262e3a7 is 50, key is test_row_0/A:col10/1732619818525/Put/seqid=0 2024-11-26T11:16:59,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742310_1486 (size=14741) 2024-11-26T11:16:59,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/307f8415f6ea41b9ad4e8e1c7262e3a7 2024-11-26T11:16:59,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/65164f10978c433498f6666a49184f0f is 50, key is test_row_0/B:col10/1732619818525/Put/seqid=0 2024-11-26T11:16:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742311_1487 (size=12301) 2024-11-26T11:16:59,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619879243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619879244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619879249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619879256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619879257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T11:16:59,323 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:16:59,323 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:16:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-26T11:16:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T11:16:59,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:16:59,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:16:59,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:16:59,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619879363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619879363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619879363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619879372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619879373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T11:16:59,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T11:16:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619879582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619879583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619879583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619879593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619879593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/65164f10978c433498f6666a49184f0f 2024-11-26T11:16:59,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/2ae11051b9744f68990da0d42df7ae81 is 50, key is test_row_0/C:col10/1732619818525/Put/seqid=0 2024-11-26T11:16:59,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742312_1488 (size=12301) 2024-11-26T11:16:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T11:16:59,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T11:16:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:59,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T11:16:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619879903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619879903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619879903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619879903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:16:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619879912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:16:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:16:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:16:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T11:16:59,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T11:16:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:16:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:16:59,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:16:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:00,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/2ae11051b9744f68990da0d42df7ae81 2024-11-26T11:17:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/307f8415f6ea41b9ad4e8e1c7262e3a7 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/307f8415f6ea41b9ad4e8e1c7262e3a7 2024-11-26T11:17:00,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/307f8415f6ea41b9ad4e8e1c7262e3a7, entries=200, sequenceid=385, filesize=14.4 K 2024-11-26T11:17:00,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/65164f10978c433498f6666a49184f0f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/65164f10978c433498f6666a49184f0f 2024-11-26T11:17:00,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/65164f10978c433498f6666a49184f0f, entries=150, sequenceid=385, filesize=12.0 K 2024-11-26T11:17:00,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/2ae11051b9744f68990da0d42df7ae81 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/2ae11051b9744f68990da0d42df7ae81 2024-11-26T11:17:00,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/2ae11051b9744f68990da0d42df7ae81, entries=150, sequenceid=385, filesize=12.0 K 2024-11-26T11:17:00,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 54ea48ed016411b0401034b85aa37157 in 847ms, sequenceid=385, compaction requested=true 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:00,036 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:00,036 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:00,037 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:00,037 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:00,037 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/A is initiating minor compaction (all files) 2024-11-26T11:17:00,037 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/B is initiating minor compaction (all files) 2024-11-26T11:17:00,037 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/B in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:00,037 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/A in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:00,037 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/a8fa5763109047c0b2836e85b84205f8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/4b864aa24ef8434abb4476a11d9bc01a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/65164f10978c433498f6666a49184f0f] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=36.7 K 2024-11-26T11:17:00,037 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/34b6525f20f7440aafe1f4a45cbb9a4a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f025d1f78e484954b35c28f09b77fe95, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/307f8415f6ea41b9ad4e8e1c7262e3a7] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=39.1 K 2024-11-26T11:17:00,037 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b6525f20f7440aafe1f4a45cbb9a4a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732619816121 2024-11-26T11:17:00,037 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting a8fa5763109047c0b2836e85b84205f8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732619816121 2024-11-26T11:17:00,038 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b864aa24ef8434abb4476a11d9bc01a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732619817325 2024-11-26T11:17:00,038 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f025d1f78e484954b35c28f09b77fe95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732619817325 2024-11-26T11:17:00,038 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 65164f10978c433498f6666a49184f0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732619818524 2024-11-26T11:17:00,038 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 307f8415f6ea41b9ad4e8e1c7262e3a7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732619818524 2024-11-26T11:17:00,043 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:00,044 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:00,044 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/72d4096823f44e1fa11e394227277c74 is 50, key is test_row_0/B:col10/1732619818525/Put/seqid=0 2024-11-26T11:17:00,044 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/6d3187f1d63545b5bcbca253b9fb70ed is 50, key is test_row_0/A:col10/1732619818525/Put/seqid=0 2024-11-26T11:17:00,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742313_1489 (size=13051) 2024-11-26T11:17:00,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742314_1490 (size=13051) 2024-11-26T11:17:00,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T11:17:00,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:00,126 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T11:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:17:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:00,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/e7a39bd127b6411da5d1d82abfa1a4d5 is 50, key is test_row_0/A:col10/1732619819248/Put/seqid=0 2024-11-26T11:17:00,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742315_1491 (size=12301) 2024-11-26T11:17:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:17:00,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. as already flushing 2024-11-26T11:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619880434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619880435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619880437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619880440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619880448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,453 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/6d3187f1d63545b5bcbca253b9fb70ed as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6d3187f1d63545b5bcbca253b9fb70ed 2024-11-26T11:17:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,457 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/A of 54ea48ed016411b0401034b85aa37157 into 6d3187f1d63545b5bcbca253b9fb70ed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:00,458 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:17:00,458 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/A, priority=13, startTime=1732619820036; duration=0sec 2024-11-26T11:17:00,458 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:00,458 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:17:00,458 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:00,458 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:00,459 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): 54ea48ed016411b0401034b85aa37157/C is initiating minor compaction (all files) 2024-11-26T11:17:00,459 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ea48ed016411b0401034b85aa37157/C in TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:00,459 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/372fdcb3675b428fa10c8246d05da2fc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4fa13b24dddf412e8c33c5e6463023eb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/2ae11051b9744f68990da0d42df7ae81] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp, totalSize=36.7 K 2024-11-26T11:17:00,459 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 372fdcb3675b428fa10c8246d05da2fc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732619816121 2024-11-26T11:17:00,459 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa13b24dddf412e8c33c5e6463023eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732619817325 2024-11-26T11:17:00,459 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae11051b9744f68990da0d42df7ae81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732619818524 2024-11-26T11:17:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T11:17:00,471 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ea48ed016411b0401034b85aa37157#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:00,471 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7d347cd008ca45f487f19ecd208c957e is 50, key is test_row_0/C:col10/1732619818525/Put/seqid=0 2024-11-26T11:17:00,474 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/72d4096823f44e1fa11e394227277c74 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/72d4096823f44e1fa11e394227277c74 2024-11-26T11:17:00,478 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/B of 54ea48ed016411b0401034b85aa37157 into 72d4096823f44e1fa11e394227277c74(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:00,478 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:17:00,478 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/B, priority=13, startTime=1732619820036; duration=0sec 2024-11-26T11:17:00,478 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:00,478 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:17:00,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742316_1492 (size=13051) 2024-11-26T11:17:00,533 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/e7a39bd127b6411da5d1d82abfa1a4d5 2024-11-26T11:17:00,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/1a9722d541e445fda9b6cd5064f09a77 is 50, key is test_row_0/B:col10/1732619819248/Put/seqid=0 2024-11-26T11:17:00,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742317_1493 (size=12301) 2024-11-26T11:17:00,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619880552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619880552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619880552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619880553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619880563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,604 INFO [Thread-1917 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1917. 2024-11-26T11:17:00,604 DEBUG [Thread-1917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:00,604 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,604 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:00,609 INFO [Thread-1915 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1915. 2024-11-26T11:17:00,610 DEBUG [Thread-1915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:00,610 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,610 INFO [Thread-1913 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1913. 2024-11-26T11:17:00,610 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,610 DEBUG [Thread-1913 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:00,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:00,610 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,610 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:00,614 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-26T11:17:00,614 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:00,614 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,614 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:00,620 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-26T11:17:00,620 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:00,620 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,620 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:00,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619880764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619880764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619880765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619880765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619880783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:00,885 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/7d347cd008ca45f487f19ecd208c957e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7d347cd008ca45f487f19ecd208c957e 2024-11-26T11:17:00,889 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ea48ed016411b0401034b85aa37157/C of 54ea48ed016411b0401034b85aa37157 into 7d347cd008ca45f487f19ecd208c957e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:00,889 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:17:00,889 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., storeName=54ea48ed016411b0401034b85aa37157/C, priority=13, startTime=1732619820036; duration=0sec 2024-11-26T11:17:00,889 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:00,889 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:17:00,943 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/1a9722d541e445fda9b6cd5064f09a77 2024-11-26T11:17:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/5166cdf933b14941aff35d3ec78e725c is 50, key is test_row_0/C:col10/1732619819248/Put/seqid=0 2024-11-26T11:17:00,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742318_1494 (size=12301) 2024-11-26T11:17:01,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36594 deadline: 1732619881073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36596 deadline: 1732619881073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36666 deadline: 1732619881073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36658 deadline: 1732619881073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:01,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36624 deadline: 1732619881093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 , the old value is region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=2 because the exception is null or not the one we care about 2024-11-26T11:17:01,360 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/5166cdf933b14941aff35d3ec78e725c 2024-11-26T11:17:01,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/e7a39bd127b6411da5d1d82abfa1a4d5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e7a39bd127b6411da5d1d82abfa1a4d5 2024-11-26T11:17:01,373 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e7a39bd127b6411da5d1d82abfa1a4d5, entries=150, sequenceid=405, filesize=12.0 K 2024-11-26T11:17:01,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/1a9722d541e445fda9b6cd5064f09a77 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/1a9722d541e445fda9b6cd5064f09a77 2024-11-26T11:17:01,378 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/1a9722d541e445fda9b6cd5064f09a77, entries=150, sequenceid=405, filesize=12.0 K 2024-11-26T11:17:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/5166cdf933b14941aff35d3ec78e725c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/5166cdf933b14941aff35d3ec78e725c 2024-11-26T11:17:01,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/5166cdf933b14941aff35d3ec78e725c, entries=150, sequenceid=405, filesize=12.0 K 2024-11-26T11:17:01,384 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54ea48ed016411b0401034b85aa37157 in 1258ms, sequenceid=405, compaction requested=false 2024-11-26T11:17:01,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:17:01,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:01,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-26T11:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-26T11:17:01,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-26T11:17:01,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0230 sec 2024-11-26T11:17:01,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.0650 sec 2024-11-26T11:17:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T11:17:01,475 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on 54ea48ed016411b0401034b85aa37157 2024-11-26T11:17:01,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T11:17:01,586 INFO [Thread-1904 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1904. 2024-11-26T11:17:01,586 DEBUG [Thread-1904 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:01,586 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,587 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:17:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:01,588 INFO [Thread-1906 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1906. 2024-11-26T11:17:01,588 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-11-26T11:17:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:17:01,588 DEBUG [Thread-1906 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:01,588 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,588 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:17:01,588 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:01,588 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:01,588 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:01,590 INFO [Thread-1902 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1902. 2024-11-26T11:17:01,590 DEBUG [Thread-1902 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:01,590 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,590 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:01,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/2ac6a254bbe94abb87959fcddd233535 is 50, key is test_row_0/A:col10/1732619821584/Put/seqid=0 2024-11-26T11:17:01,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742319_1495 (size=12301) 2024-11-26T11:17:01,604 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-11-26T11:17:01,604 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:01,604 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,604 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1209 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3627 rows 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1218 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3654 rows 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1212 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3636 rows 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1218 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3654 rows 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-11-26T11:17:01,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3667 rows 2024-11-26T11:17:01,605 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:17:01,605 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:17:01,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:01,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:01,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T11:17:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:01,611 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619821611"}]},"ts":"1732619821611"} 2024-11-26T11:17:01,613 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T11:17:01,613 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T11:17:01,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:17:01,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, UNASSIGN}] 2024-11-26T11:17:01,616 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, UNASSIGN 2024-11-26T11:17:01,616 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=54ea48ed016411b0401034b85aa37157, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:01,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, UNASSIGN because future has completed 2024-11-26T11:17:01,618 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:17:01,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:17:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:01,770 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 54ea48ed016411b0401034b85aa37157 2024-11-26T11:17:01,770 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:17:01,770 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 54ea48ed016411b0401034b85aa37157, disabling compactions & flushes 2024-11-26T11:17:01,770 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:01,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/2ac6a254bbe94abb87959fcddd233535 2024-11-26T11:17:02,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/54b63fc5c4614c5c8844380f85b6f08c is 50, key is test_row_0/B:col10/1732619821584/Put/seqid=0 2024-11-26T11:17:02,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742320_1496 (size=12301) 2024-11-26T11:17:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:02,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/54b63fc5c4614c5c8844380f85b6f08c 2024-11-26T11:17:02,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/bc3f4bde0d824159a3bdc2c14ec42fe6 is 50, key is test_row_0/C:col10/1732619821584/Put/seqid=0 2024-11-26T11:17:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742321_1497 (size=12301) 2024-11-26T11:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:02,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/bc3f4bde0d824159a3bdc2c14ec42fe6 2024-11-26T11:17:02,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/2ac6a254bbe94abb87959fcddd233535 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2ac6a254bbe94abb87959fcddd233535 2024-11-26T11:17:02,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2ac6a254bbe94abb87959fcddd233535, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T11:17:02,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/54b63fc5c4614c5c8844380f85b6f08c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/54b63fc5c4614c5c8844380f85b6f08c 2024-11-26T11:17:02,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/54b63fc5c4614c5c8844380f85b6f08c, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T11:17:02,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/bc3f4bde0d824159a3bdc2c14ec42fe6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/bc3f4bde0d824159a3bdc2c14ec42fe6 2024-11-26T11:17:02,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/bc3f4bde0d824159a3bdc2c14ec42fe6, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T11:17:02,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 54ea48ed016411b0401034b85aa37157 in 1263ms, sequenceid=423, compaction requested=true 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ea48ed016411b0401034b85aa37157: 2024-11-26T11:17:02,850 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:02,850 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. because compaction request was cancelled 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. after waiting 0 ms 2024-11-26T11:17:02,850 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:A 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:02,850 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. because compaction request was cancelled 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:B, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:02,850 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:B 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ea48ed016411b0401034b85aa37157:C, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:02,850 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 54ea48ed016411b0401034b85aa37157 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-26T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:02,850 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. because compaction request was cancelled 2024-11-26T11:17:02,850 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ea48ed016411b0401034b85aa37157:C 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=A 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=B 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54ea48ed016411b0401034b85aa37157, store=C 2024-11-26T11:17:02,850 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:02,854 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/ab5a86fa060640eab34d6e83c990984f is 50, key is test_row_1/A:col10/1732619821603/Put/seqid=0 2024-11-26T11:17:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742322_1498 (size=7415) 2024-11-26T11:17:03,259 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/ab5a86fa060640eab34d6e83c990984f 2024-11-26T11:17:03,272 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/2e676132ce0248a29d6c245ae01d3929 is 50, key is test_row_1/B:col10/1732619821603/Put/seqid=0 2024-11-26T11:17:03,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742323_1499 (size=7415) 2024-11-26T11:17:03,276 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/2e676132ce0248a29d6c245ae01d3929 2024-11-26T11:17:03,283 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b23153777fee4997a407128e4e10e94b is 50, key is test_row_1/C:col10/1732619821603/Put/seqid=0 2024-11-26T11:17:03,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742324_1500 (size=7415) 2024-11-26T11:17:03,688 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b23153777fee4997a407128e4e10e94b 2024-11-26T11:17:03,698 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/A/ab5a86fa060640eab34d6e83c990984f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/ab5a86fa060640eab34d6e83c990984f 2024-11-26T11:17:03,704 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/ab5a86fa060640eab34d6e83c990984f, entries=50, sequenceid=428, filesize=7.2 K 2024-11-26T11:17:03,705 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/B/2e676132ce0248a29d6c245ae01d3929 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/2e676132ce0248a29d6c245ae01d3929 2024-11-26T11:17:03,710 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/2e676132ce0248a29d6c245ae01d3929, entries=50, sequenceid=428, filesize=7.2 K 2024-11-26T11:17:03,711 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/.tmp/C/b23153777fee4997a407128e4e10e94b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b23153777fee4997a407128e4e10e94b 2024-11-26T11:17:03,715 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b23153777fee4997a407128e4e10e94b, entries=50, sequenceid=428, filesize=7.2 K 2024-11-26T11:17:03,716 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 54ea48ed016411b0401034b85aa37157 in 866ms, sequenceid=428, compaction requested=true 2024-11-26T11:17:03,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d6e950503a0b4231ab4ce47a02bcc74e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/31f6dff9eed44da8b0ed4bdecbaf01f8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e957acdc71374eb6b45ac928ee636384, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2959c18bdea74dcf90a3233c41dafdee, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b7303d7b4da34aa4ad1b92444f608202, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/0284ef1bf29143f19d78d0b1ff2b9752, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/be90618f41344d05af2d76b695dccfcd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6f20293e8d76407ca106dcf248c3aa25, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/fd1ddb3b4e5846b6a951451587b4ad2d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/529e54aac29844f9946b168a8965b066, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/42513eb1bf464db7893ac0262ceaca5b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f061ea892be44b88889f1dbcb94af29e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/110e1ad002194d2887662113a7b044ac, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d1e38bfa8d164e80898483fa6165f52c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b1d7f145c51a422bb6d374a2a15b7dc4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/563f8c92f6e044298dbbc42aac407d84, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/48c2c48bd9314d8089da506577a0185f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/12f451180e704ccc9b2a4f5cd3bb1f54, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/22332123240b4ddaa7d2781852cd1cf8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/34b6525f20f7440aafe1f4a45cbb9a4a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f025d1f78e484954b35c28f09b77fe95, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/307f8415f6ea41b9ad4e8e1c7262e3a7] to archive 2024-11-26T11:17:03,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:17:03,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d6e950503a0b4231ab4ce47a02bcc74e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d6e950503a0b4231ab4ce47a02bcc74e 2024-11-26T11:17:03,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/31f6dff9eed44da8b0ed4bdecbaf01f8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/31f6dff9eed44da8b0ed4bdecbaf01f8 2024-11-26T11:17:03,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e957acdc71374eb6b45ac928ee636384 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e957acdc71374eb6b45ac928ee636384 2024-11-26T11:17:03,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2959c18bdea74dcf90a3233c41dafdee to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2959c18bdea74dcf90a3233c41dafdee 2024-11-26T11:17:03,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b7303d7b4da34aa4ad1b92444f608202 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b7303d7b4da34aa4ad1b92444f608202 2024-11-26T11:17:03,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/0284ef1bf29143f19d78d0b1ff2b9752 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/0284ef1bf29143f19d78d0b1ff2b9752 2024-11-26T11:17:03,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/be90618f41344d05af2d76b695dccfcd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/be90618f41344d05af2d76b695dccfcd 2024-11-26T11:17:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6f20293e8d76407ca106dcf248c3aa25 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6f20293e8d76407ca106dcf248c3aa25 2024-11-26T11:17:03,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/fd1ddb3b4e5846b6a951451587b4ad2d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/fd1ddb3b4e5846b6a951451587b4ad2d 2024-11-26T11:17:03,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/529e54aac29844f9946b168a8965b066 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/529e54aac29844f9946b168a8965b066 2024-11-26T11:17:03,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/42513eb1bf464db7893ac0262ceaca5b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/42513eb1bf464db7893ac0262ceaca5b 2024-11-26T11:17:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f061ea892be44b88889f1dbcb94af29e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f061ea892be44b88889f1dbcb94af29e 2024-11-26T11:17:03,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/110e1ad002194d2887662113a7b044ac to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/110e1ad002194d2887662113a7b044ac 2024-11-26T11:17:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d1e38bfa8d164e80898483fa6165f52c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/d1e38bfa8d164e80898483fa6165f52c 2024-11-26T11:17:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b1d7f145c51a422bb6d374a2a15b7dc4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/b1d7f145c51a422bb6d374a2a15b7dc4 2024-11-26T11:17:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/563f8c92f6e044298dbbc42aac407d84 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/563f8c92f6e044298dbbc42aac407d84 2024-11-26T11:17:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/48c2c48bd9314d8089da506577a0185f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/48c2c48bd9314d8089da506577a0185f 2024-11-26T11:17:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/12f451180e704ccc9b2a4f5cd3bb1f54 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/12f451180e704ccc9b2a4f5cd3bb1f54 2024-11-26T11:17:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/22332123240b4ddaa7d2781852cd1cf8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/22332123240b4ddaa7d2781852cd1cf8 2024-11-26T11:17:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/34b6525f20f7440aafe1f4a45cbb9a4a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/34b6525f20f7440aafe1f4a45cbb9a4a 2024-11-26T11:17:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f025d1f78e484954b35c28f09b77fe95 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/f025d1f78e484954b35c28f09b77fe95 2024-11-26T11:17:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/307f8415f6ea41b9ad4e8e1c7262e3a7 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/307f8415f6ea41b9ad4e8e1c7262e3a7 2024-11-26T11:17:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/288d780fcf5c4a87956aa63c75d5bd49, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/bbd0b17279af4239a6cb716042d664b2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/e569db345c564f698996ce6e1a4ec11a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/df5b16dfd9d84e68877ff2362f738b67, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4a7ac680bbb4c0aaa662911a8ca8db3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6eab571acb09467981ef5983319c481f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9268fefd39f741b3b3691078e98a5877, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6e591eb49b3749c7865ad67e519ed362, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/ee5f9b1f49524f00904ba275db757d91, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/cc86308650334564a3d3e360cdedbe56, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d41528cc44af497394a90cfd79997445, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/379858e8162042858a0498d3aae69032, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6753596f93454e2792571c74352f5811, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/586dc568c700405e99533854560da5a5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/32c4147e3b08449f85b6b75ad00b8daf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/169be7530fd44ee482cf2230c82065b0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9e29e3bfe07046f8ab662195ee7f55fd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4ff51c9568243e49f278ee8f8e199c5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/a8fa5763109047c0b2836e85b84205f8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/5e408d09e43e42019f7702383239e47d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/4b864aa24ef8434abb4476a11d9bc01a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/65164f10978c433498f6666a49184f0f] to archive 2024-11-26T11:17:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:17:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/288d780fcf5c4a87956aa63c75d5bd49 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/288d780fcf5c4a87956aa63c75d5bd49 2024-11-26T11:17:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/bbd0b17279af4239a6cb716042d664b2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/bbd0b17279af4239a6cb716042d664b2 2024-11-26T11:17:03,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/e569db345c564f698996ce6e1a4ec11a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/e569db345c564f698996ce6e1a4ec11a 2024-11-26T11:17:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/df5b16dfd9d84e68877ff2362f738b67 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/df5b16dfd9d84e68877ff2362f738b67 2024-11-26T11:17:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4a7ac680bbb4c0aaa662911a8ca8db3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4a7ac680bbb4c0aaa662911a8ca8db3 2024-11-26T11:17:03,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6eab571acb09467981ef5983319c481f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6eab571acb09467981ef5983319c481f 2024-11-26T11:17:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9268fefd39f741b3b3691078e98a5877 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9268fefd39f741b3b3691078e98a5877 2024-11-26T11:17:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6e591eb49b3749c7865ad67e519ed362 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6e591eb49b3749c7865ad67e519ed362 2024-11-26T11:17:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/ee5f9b1f49524f00904ba275db757d91 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/ee5f9b1f49524f00904ba275db757d91 2024-11-26T11:17:03,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/cc86308650334564a3d3e360cdedbe56 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/cc86308650334564a3d3e360cdedbe56 2024-11-26T11:17:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d41528cc44af497394a90cfd79997445 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d41528cc44af497394a90cfd79997445 2024-11-26T11:17:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/379858e8162042858a0498d3aae69032 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/379858e8162042858a0498d3aae69032 2024-11-26T11:17:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6753596f93454e2792571c74352f5811 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/6753596f93454e2792571c74352f5811 2024-11-26T11:17:03,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/586dc568c700405e99533854560da5a5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/586dc568c700405e99533854560da5a5 2024-11-26T11:17:03,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/32c4147e3b08449f85b6b75ad00b8daf to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/32c4147e3b08449f85b6b75ad00b8daf 2024-11-26T11:17:03,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/169be7530fd44ee482cf2230c82065b0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/169be7530fd44ee482cf2230c82065b0 2024-11-26T11:17:03,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9e29e3bfe07046f8ab662195ee7f55fd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/9e29e3bfe07046f8ab662195ee7f55fd 2024-11-26T11:17:03,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4ff51c9568243e49f278ee8f8e199c5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/d4ff51c9568243e49f278ee8f8e199c5 2024-11-26T11:17:03,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/a8fa5763109047c0b2836e85b84205f8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/a8fa5763109047c0b2836e85b84205f8 2024-11-26T11:17:03,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/5e408d09e43e42019f7702383239e47d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/5e408d09e43e42019f7702383239e47d 2024-11-26T11:17:03,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/4b864aa24ef8434abb4476a11d9bc01a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/4b864aa24ef8434abb4476a11d9bc01a 2024-11-26T11:17:03,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/65164f10978c433498f6666a49184f0f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/65164f10978c433498f6666a49184f0f 2024-11-26T11:17:03,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/30a79fc08fe443eaa05e0baf29a7334b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7afc51b52c704712ae1720ea6839a0ff, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7484c3334bd24a28a0ab00975d58f814, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0dde918b8c0a4b6c9c5b625a58ed1b7b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4eb022c16ffc42b5820f2144280aa4c9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b8c06dc354d240c0bf08984c8e878120, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/449dd0be86f24448bca047c7187697b4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/24e38755ee274652b2319fa47a3d5c23, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/fade5d5c372f412c80d3140105cdae64, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/a74f085b89414736bf284112d69369e9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0cc82d105acc40e99e9a7fc29075661d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1a17553e3e834883b192d678dfe27b06, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b6b2761aab404237acec6b38b2b063d4, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1936d82972414cea97041195628c963d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4477ad839a8e4f8b8056809b4255c64c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/d5ea4f826c45489aa3df8415d29e11dd, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/3c7a4dfb5edb41bc955ace612149ad0a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/62c4d32178be4ebe833918e6d5ddf3f8, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/372fdcb3675b428fa10c8246d05da2fc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/42dc8f210f154bee998c9dd30307847c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4fa13b24dddf412e8c33c5e6463023eb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/2ae11051b9744f68990da0d42df7ae81] to archive 2024-11-26T11:17:03,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:17:03,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/30a79fc08fe443eaa05e0baf29a7334b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/30a79fc08fe443eaa05e0baf29a7334b 2024-11-26T11:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:03,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7afc51b52c704712ae1720ea6839a0ff to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7afc51b52c704712ae1720ea6839a0ff 2024-11-26T11:17:03,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7484c3334bd24a28a0ab00975d58f814 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7484c3334bd24a28a0ab00975d58f814 2024-11-26T11:17:03,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0dde918b8c0a4b6c9c5b625a58ed1b7b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0dde918b8c0a4b6c9c5b625a58ed1b7b 2024-11-26T11:17:03,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4eb022c16ffc42b5820f2144280aa4c9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4eb022c16ffc42b5820f2144280aa4c9 2024-11-26T11:17:03,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b8c06dc354d240c0bf08984c8e878120 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b8c06dc354d240c0bf08984c8e878120 2024-11-26T11:17:03,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/449dd0be86f24448bca047c7187697b4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/449dd0be86f24448bca047c7187697b4 2024-11-26T11:17:03,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/24e38755ee274652b2319fa47a3d5c23 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/24e38755ee274652b2319fa47a3d5c23 2024-11-26T11:17:03,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/fade5d5c372f412c80d3140105cdae64 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/fade5d5c372f412c80d3140105cdae64 2024-11-26T11:17:03,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/a74f085b89414736bf284112d69369e9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/a74f085b89414736bf284112d69369e9 2024-11-26T11:17:03,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0cc82d105acc40e99e9a7fc29075661d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/0cc82d105acc40e99e9a7fc29075661d 2024-11-26T11:17:03,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1a17553e3e834883b192d678dfe27b06 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1a17553e3e834883b192d678dfe27b06 2024-11-26T11:17:03,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b6b2761aab404237acec6b38b2b063d4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b6b2761aab404237acec6b38b2b063d4 2024-11-26T11:17:03,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1936d82972414cea97041195628c963d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/1936d82972414cea97041195628c963d 2024-11-26T11:17:03,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4477ad839a8e4f8b8056809b4255c64c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4477ad839a8e4f8b8056809b4255c64c 2024-11-26T11:17:03,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/d5ea4f826c45489aa3df8415d29e11dd to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/d5ea4f826c45489aa3df8415d29e11dd 2024-11-26T11:17:03,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/3c7a4dfb5edb41bc955ace612149ad0a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/3c7a4dfb5edb41bc955ace612149ad0a 2024-11-26T11:17:03,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/62c4d32178be4ebe833918e6d5ddf3f8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/62c4d32178be4ebe833918e6d5ddf3f8 2024-11-26T11:17:03,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/372fdcb3675b428fa10c8246d05da2fc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/372fdcb3675b428fa10c8246d05da2fc 2024-11-26T11:17:03,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/42dc8f210f154bee998c9dd30307847c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/42dc8f210f154bee998c9dd30307847c 2024-11-26T11:17:03,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4fa13b24dddf412e8c33c5e6463023eb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/4fa13b24dddf412e8c33c5e6463023eb 2024-11-26T11:17:03,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/2ae11051b9744f68990da0d42df7ae81 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/2ae11051b9744f68990da0d42df7ae81 2024-11-26T11:17:03,780 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=1 2024-11-26T11:17:03,781 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. 2024-11-26T11:17:03,781 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 54ea48ed016411b0401034b85aa37157: Waiting for close lock at 1732619821770Running coprocessor pre-close hooks at 1732619821770Disabling compacts and flushes for region at 1732619821770Disabling writes for close at 1732619822850 (+1080 ms)Obtaining lock to block concurrent updates at 1732619822850Preparing flush snapshotting stores in 54ea48ed016411b0401034b85aa37157 at 1732619822850Finished memstore snapshotting TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732619822850Flushing stores of TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157. at 1732619822851 (+1 ms)Flushing 54ea48ed016411b0401034b85aa37157/A: creating writer at 1732619822851Flushing 54ea48ed016411b0401034b85aa37157/A: appending metadata at 1732619822853 (+2 ms)Flushing 54ea48ed016411b0401034b85aa37157/A: closing flushed file at 1732619822853Flushing 54ea48ed016411b0401034b85aa37157/B: creating writer at 1732619823268 (+415 ms)Flushing 54ea48ed016411b0401034b85aa37157/B: appending metadata at 1732619823271 (+3 ms)Flushing 54ea48ed016411b0401034b85aa37157/B: closing flushed file at 1732619823271Flushing 54ea48ed016411b0401034b85aa37157/C: creating writer at 1732619823279 (+8 ms)Flushing 54ea48ed016411b0401034b85aa37157/C: appending metadata at 1732619823282 (+3 ms)Flushing 54ea48ed016411b0401034b85aa37157/C: closing flushed file at 1732619823282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a167598: reopening flushed file at 1732619823696 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53fe27b4: reopening flushed file at 1732619823704 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40fd0524: reopening flushed file at 1732619823710 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 54ea48ed016411b0401034b85aa37157 in 866ms, sequenceid=428, compaction requested=true at 1732619823716 (+6 ms)Writing region close event to WAL at 1732619823778 (+62 ms)Running coprocessor post-close hooks at 1732619823781 (+3 ms)Closed at 1732619823781 2024-11-26T11:17:03,783 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 54ea48ed016411b0401034b85aa37157 2024-11-26T11:17:03,783 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=54ea48ed016411b0401034b85aa37157, regionState=CLOSED 2024-11-26T11:17:03,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:17:03,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-26T11:17:03,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 54ea48ed016411b0401034b85aa37157, server=7f3a3edaecc3,37499,1732619676667 in 2.1670 sec 2024-11-26T11:17:03,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-26T11:17:03,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ea48ed016411b0401034b85aa37157, UNASSIGN in 2.1720 sec 2024-11-26T11:17:03,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-26T11:17:03,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1760 sec 2024-11-26T11:17:03,793 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619823793"}]},"ts":"1732619823793"} 2024-11-26T11:17:03,795 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T11:17:03,795 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T11:17:03,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1890 sec 2024-11-26T11:17:04,264 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T11:17:05,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T11:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T11:17:05,785 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:05,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T11:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,791 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T11:17:05,793 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,796 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157 2024-11-26T11:17:05,799 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/recovered.edits] 2024-11-26T11:17:05,802 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2ac6a254bbe94abb87959fcddd233535 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/2ac6a254bbe94abb87959fcddd233535 2024-11-26T11:17:05,804 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6d3187f1d63545b5bcbca253b9fb70ed to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/6d3187f1d63545b5bcbca253b9fb70ed 2024-11-26T11:17:05,805 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/ab5a86fa060640eab34d6e83c990984f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/ab5a86fa060640eab34d6e83c990984f 2024-11-26T11:17:05,807 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e7a39bd127b6411da5d1d82abfa1a4d5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/A/e7a39bd127b6411da5d1d82abfa1a4d5 2024-11-26T11:17:05,810 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/1a9722d541e445fda9b6cd5064f09a77 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/1a9722d541e445fda9b6cd5064f09a77 2024-11-26T11:17:05,811 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/2e676132ce0248a29d6c245ae01d3929 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/2e676132ce0248a29d6c245ae01d3929 2024-11-26T11:17:05,813 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/54b63fc5c4614c5c8844380f85b6f08c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/54b63fc5c4614c5c8844380f85b6f08c 2024-11-26T11:17:05,814 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/72d4096823f44e1fa11e394227277c74 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/B/72d4096823f44e1fa11e394227277c74 2024-11-26T11:17:05,817 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/5166cdf933b14941aff35d3ec78e725c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/5166cdf933b14941aff35d3ec78e725c 2024-11-26T11:17:05,819 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7d347cd008ca45f487f19ecd208c957e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/7d347cd008ca45f487f19ecd208c957e 2024-11-26T11:17:05,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b23153777fee4997a407128e4e10e94b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/b23153777fee4997a407128e4e10e94b 2024-11-26T11:17:05,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/bc3f4bde0d824159a3bdc2c14ec42fe6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/C/bc3f4bde0d824159a3bdc2c14ec42fe6 2024-11-26T11:17:05,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/recovered.edits/431.seqid to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157/recovered.edits/431.seqid 2024-11-26T11:17:05,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/54ea48ed016411b0401034b85aa37157 2024-11-26T11:17:05,826 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T11:17:05,829 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,832 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T11:17:05,834 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T11:17:05,836 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,836 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T11:17:05,836 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732619825836"}]},"ts":"9223372036854775807"} 2024-11-26T11:17:05,839 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T11:17:05,839 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 54ea48ed016411b0401034b85aa37157, NAME => 'TestAcidGuarantees,,1732619798373.54ea48ed016411b0401034b85aa37157.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T11:17:05,839 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T11:17:05,839 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732619825839"}]},"ts":"9223372036854775807"} 2024-11-26T11:17:05,841 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T11:17:05,842 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-26T11:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T11:17:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T11:17:05,907 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:05,914 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 470), ProcessCount=11 (was 11), AvailableMemoryMB=2681 (was 2678) - AvailableMemoryMB LEAK? - 2024-11-26T11:17:05,923 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=2681 2024-11-26T11:17:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:17:05,925 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:17:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:05,926 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T11:17:05,926 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:05,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-26T11:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T11:17:05,927 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T11:17:05,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742325_1501 (size=1061) 2024-11-26T11:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T11:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T11:17:06,334 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29 2024-11-26T11:17:06,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742326_1502 (size=53) 2024-11-26T11:17:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T11:17:06,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:17:06,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fa0c0e175d4b592132f0942e2a244b29, disabling compactions & flushes 2024-11-26T11:17:06,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:06,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:06,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. after waiting 0 ms 2024-11-26T11:17:06,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:06,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:06,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fa0c0e175d4b592132f0942e2a244b29: Waiting for close lock at 1732619826739Disabling compacts and flushes for region at 1732619826739Disabling writes for close at 1732619826739Writing region close event to WAL at 1732619826739Closed at 1732619826739 2024-11-26T11:17:06,740 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T11:17:06,740 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732619826740"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732619826740"}]},"ts":"1732619826740"} 2024-11-26T11:17:06,742 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T11:17:06,742 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T11:17:06,742 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619826742"}]},"ts":"1732619826742"} 2024-11-26T11:17:06,743 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T11:17:06,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, ASSIGN}] 2024-11-26T11:17:06,744 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, ASSIGN 2024-11-26T11:17:06,745 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, ASSIGN; state=OFFLINE, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=false 2024-11-26T11:17:06,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:06,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, ASSIGN because future has completed 2024-11-26T11:17:06,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:17:07,051 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:07,052 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:17:07,052 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,052 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:17:07,052 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,052 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,053 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,054 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:17:07,054 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa0c0e175d4b592132f0942e2a244b29 columnFamilyName A 2024-11-26T11:17:07,054 DEBUG [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:07,055 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(327): Store=fa0c0e175d4b592132f0942e2a244b29/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:17:07,055 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,056 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:17:07,056 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa0c0e175d4b592132f0942e2a244b29 columnFamilyName B 2024-11-26T11:17:07,056 DEBUG [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:07,056 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(327): Store=fa0c0e175d4b592132f0942e2a244b29/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:17:07,056 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,057 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:17:07,057 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa0c0e175d4b592132f0942e2a244b29 columnFamilyName C 2024-11-26T11:17:07,057 DEBUG [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:07,058 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(327): Store=fa0c0e175d4b592132f0942e2a244b29/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:17:07,058 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:07,058 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,058 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,059 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,059 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,060 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,060 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:17:07,061 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,062 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T11:17:07,063 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened fa0c0e175d4b592132f0942e2a244b29; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62376555, jitterRate=-0.07051689922809601}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:17:07,063 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T11:17:07,063 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for fa0c0e175d4b592132f0942e2a244b29: Running coprocessor pre-open hook at 1732619827052Writing region info on filesystem at 1732619827052Initializing all the Stores at 1732619827053 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619827053Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619827053Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619827053Cleaning up temporary data from old regions at 1732619827060 (+7 ms)Running coprocessor post-open hooks at 1732619827063 (+3 ms)Region opened successfully at 1732619827063 2024-11-26T11:17:07,064 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., pid=134, masterSystemTime=1732619827049 2024-11-26T11:17:07,065 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:07,065 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:07,066 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=OPEN, openSeqNum=2, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:07,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:17:07,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-26T11:17:07,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 in 171 msec 2024-11-26T11:17:07,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-26T11:17:07,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, ASSIGN in 325 msec 2024-11-26T11:17:07,071 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T11:17:07,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619827071"}]},"ts":"1732619827071"} 2024-11-26T11:17:07,072 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T11:17:07,073 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T11:17:07,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-26T11:17:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T11:17:08,073 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:08,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:08,074 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:08,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:08,075 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540c6102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:08,077 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e25a7dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:08,078 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:08,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:08,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T11:17:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7f3a3edaecc3,42255,1732619675105 2024-11-26T11:17:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5dcee790 2024-11-26T11:17:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T11:17:08,084 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T11:17:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T11:17:08,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T11:17:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:08,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742327_1503 (size=1097) 2024-11-26T11:17:08,501 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T11:17:08,501 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T11:17:08,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:17:08,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, REOPEN/MOVE}] 2024-11-26T11:17:08,509 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, REOPEN/MOVE 2024-11-26T11:17:08,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:08,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, REOPEN/MOVE because future has completed 2024-11-26T11:17:08,511 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:17:08,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:17:08,664 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,664 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:17:08,665 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing fa0c0e175d4b592132f0942e2a244b29, disabling compactions & flushes 2024-11-26T11:17:08,665 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:08,665 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:08,665 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. after waiting 0 ms 2024-11-26T11:17:08,665 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:08,669 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T11:17:08,670 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:08,670 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for fa0c0e175d4b592132f0942e2a244b29: Waiting for close lock at 1732619828665Running coprocessor pre-close hooks at 1732619828665Disabling compacts and flushes for region at 1732619828665Disabling writes for close at 1732619828665Writing region close event to WAL at 1732619828665Running coprocessor post-close hooks at 1732619828670 (+5 ms)Closed at 1732619828670 2024-11-26T11:17:08,670 WARN [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: fa0c0e175d4b592132f0942e2a244b29 to self. 2024-11-26T11:17:08,672 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=CLOSED 2024-11-26T11:17:08,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:17:08,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-26T11:17:08,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 in 162 msec 2024-11-26T11:17:08,676 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, REOPEN/MOVE; state=CLOSED, location=7f3a3edaecc3,37499,1732619676667; forceNewPlan=false, retain=true 2024-11-26T11:17:08,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=OPENING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:08,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, REOPEN/MOVE because future has completed 2024-11-26T11:17:08,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:17:08,984 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:08,985 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} 2024-11-26T11:17:08,985 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,985 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T11:17:08,985 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,985 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,987 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,988 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:17:08,988 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa0c0e175d4b592132f0942e2a244b29 columnFamilyName A 2024-11-26T11:17:08,990 DEBUG [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:08,990 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(327): Store=fa0c0e175d4b592132f0942e2a244b29/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:17:08,991 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,991 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:17:08,992 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa0c0e175d4b592132f0942e2a244b29 columnFamilyName B 2024-11-26T11:17:08,992 DEBUG [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:08,992 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(327): Store=fa0c0e175d4b592132f0942e2a244b29/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:17:08,992 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,993 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T11:17:08,993 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa0c0e175d4b592132f0942e2a244b29 columnFamilyName C 2024-11-26T11:17:08,993 DEBUG [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:08,994 INFO [StoreOpener-fa0c0e175d4b592132f0942e2a244b29-1 {}] regionserver.HStore(327): Store=fa0c0e175d4b592132f0942e2a244b29/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T11:17:08,994 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:08,994 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,995 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,996 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,997 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,997 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:08,998 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T11:17:08,999 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:09,000 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened fa0c0e175d4b592132f0942e2a244b29; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72137590, jitterRate=0.07493385672569275}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T11:17:09,000 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:09,001 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for fa0c0e175d4b592132f0942e2a244b29: Running coprocessor pre-open hook at 1732619828985Writing region info on filesystem at 1732619828985Initializing all the Stores at 1732619828987 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619828987Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619828987Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732619828987Cleaning up temporary data from old regions at 1732619828997 (+10 ms)Running coprocessor post-open hooks at 1732619829000 (+3 ms)Region opened successfully at 1732619829001 (+1 ms) 2024-11-26T11:17:09,002 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., pid=139, masterSystemTime=1732619828981 2024-11-26T11:17:09,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=OPEN, openSeqNum=5, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,013 DEBUG [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:17:09,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-26T11:17:09,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 in 186 msec 2024-11-26T11:17:09,017 INFO [RS_OPEN_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-26T11:17:09,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, REOPEN/MOVE in 510 msec 2024-11-26T11:17:09,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-26T11:17:09,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-26T11:17:09,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-26T11:17:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T11:17:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T11:17:09,133 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:09,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,135 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc479d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,137 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25546afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,140 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61340d03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2772f526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c19e3f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,147 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed94398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,149 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165d978e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,151 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbf516d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,156 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193c01d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c083e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7bd325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c4f31c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,165 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d3102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,167 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5174aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,173 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd7f90a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,175 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747fcdd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,179 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d5d76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,181 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2721a4ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7f3a3edaecc3,42255,-1 for getting cluster id 2024-11-26T11:17:09,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T11:17:09,188 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4be5fd21-3c4c-4c45-86ec-49ce10916dea' 2024-11-26T11:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T11:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4be5fd21-3c4c-4c45-86ec-49ce10916dea" 2024-11-26T11:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbc7f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7f3a3edaecc3,42255,-1] 2024-11-26T11:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T11:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:09,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T11:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c4ef49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T11:17:09,192 DEBUG [Thread-2263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,196 DEBUG [Thread-2255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,197 DEBUG [Thread-2255 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,198 DEBUG [Thread-2255 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,211 DEBUG [Thread-2261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:09,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:17:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:09,213 DEBUG [Thread-2261 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,213 DEBUG [Thread-2261 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,216 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-26T11:17:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:09,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:09,219 DEBUG [Thread-2274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,220 DEBUG [Thread-2274 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:09,221 DEBUG [Thread-2274 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:09,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,226 DEBUG [Thread-2257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,248 DEBUG [Thread-2259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619829240, max timestamp=1732619829246Num uniques 859; 2024-11-26T11:17:09,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619829240, max timestamp=1732619829246Num uniques 859; 2024-11-26T11:17:09,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:09,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:09,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732619829240, max timestamp=1732619829246Num uniques 859; 2024-11-26T11:17:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T11:17:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,256 DEBUG [Thread-2270 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,265 DEBUG [Thread-2268 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,266 DEBUG [Thread-2268 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,266 DEBUG [Thread-2268 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,279 DEBUG [Thread-2266 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619889279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619889279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619889286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732619829275, max timestamp=1732619829285Num uniques 1009; 2024-11-26T11:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-26T11:17:09,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619829275, max timestamp=1732619829285Num uniques 1018; 2024-11-26T11:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619829275, max timestamp=1732619829285Num uniques 1018; 2024-11-26T11:17:09,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:17:09,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e81fb5e830e446ba3f1cf9c4aeeb71a_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619829210/Put/seqid=0 2024-11-26T11:17:09,297 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:17:09,304 DEBUG [Thread-2272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T11:17:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7f3a3edaecc3,37499,1732619676667, seqNum=-1] 2024-11-26T11:17:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T11:17:09,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T11:17:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5] 2024-11-26T11:17:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:09,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742328_1504 (size=12154) 2024-11-26T11:17:09,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:09,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e81fb5e830e446ba3f1cf9c4aeeb71a_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e81fb5e830e446ba3f1cf9c4aeeb71a_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:09,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/4a6a9e9c52b64387a9fc2300ebc94194, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:09,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/4a6a9e9c52b64387a9fc2300ebc94194 is 175, key is test_row_0/A:col10/1732619829210/Put/seqid=0 2024-11-26T11:17:09,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742329_1505 (size=30955) 2024-11-26T11:17:09,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619889395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619889403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619889405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619889406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619889415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619889513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619889523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619889523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:09,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:09,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619889533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619889624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619889723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619889733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619889743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619889752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/4a6a9e9c52b64387a9fc2300ebc94194 2024-11-26T11:17:09,785 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T11:17:09,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/53b1954aec65410e8abae7b43cf38570 is 50, key is test_row_0/B:col10/1732619829210/Put/seqid=0 2024-11-26T11:17:09,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:09,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:09,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:09,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742330_1506 (size=12001) 2024-11-26T11:17:09,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/53b1954aec65410e8abae7b43cf38570 2024-11-26T11:17:09,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/32388a2d00324c56ada47cc15c7b6374 is 50, key is test_row_0/C:col10/1732619829210/Put/seqid=0 2024-11-26T11:17:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619889932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:09,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:09,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742331_1507 (size=12001) 2024-11-26T11:17:10,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619890033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619890043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619890053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619890072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/32388a2d00324c56ada47cc15c7b6374 2024-11-26T11:17:10,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/4a6a9e9c52b64387a9fc2300ebc94194 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194 2024-11-26T11:17:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:10,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194, entries=150, sequenceid=14, filesize=30.2 K 2024-11-26T11:17:10,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/53b1954aec65410e8abae7b43cf38570 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/53b1954aec65410e8abae7b43cf38570 2024-11-26T11:17:10,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/53b1954aec65410e8abae7b43cf38570, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T11:17:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/32388a2d00324c56ada47cc15c7b6374 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/32388a2d00324c56ada47cc15c7b6374 2024-11-26T11:17:10,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/32388a2d00324c56ada47cc15c7b6374, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T11:17:10,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.85 KB/141162 for fa0c0e175d4b592132f0942e2a244b29 in 1167ms, sequenceid=14, compaction requested=false 2024-11-26T11:17:10,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:10,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732619829384, max timestamp=1732619830445Num uniques 1077; 2024-11-26T11:17:10,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732619829384, max timestamp=1732619830445Num uniques 1077; 2024-11-26T11:17:10,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T11:17:10,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T11:17:10,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732619829384, max timestamp=1732619830445Num uniques 1009; 2024-11-26T11:17:10,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-11-26T11:17:10,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=26.84 KB heapSize=64.55 KB 2024-11-26T11:17:10,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:10,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112653b044cf74eb46609db96bf16225ed01_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619829404/Put/seqid=0 2024-11-26T11:17:10,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742332_1508 (size=12154) 2024-11-26T11:17:10,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:10,532 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112653b044cf74eb46609db96bf16225ed01_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653b044cf74eb46609db96bf16225ed01_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:10,533 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d8f15cbe89a54b7bab5aa69d8943c17c, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:10,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d8f15cbe89a54b7bab5aa69d8943c17c is 175, key is test_row_0/A:col10/1732619829404/Put/seqid=0 2024-11-26T11:17:10,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619830531, max timestamp=1732619830546Num uniques 859; 2024-11-26T11:17:10,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:10,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619830531, max timestamp=1732619830546Num uniques 859; 2024-11-26T11:17:10,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:10,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619830531, max timestamp=1732619830546Num uniques 859; 2024-11-26T11:17:10,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:10,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742333_1509 (size=30955) 2024-11-26T11:17:10,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d8f15cbe89a54b7bab5aa69d8943c17c 2024-11-26T11:17:10,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/409d2818fd174d6ea2f8bb53ad3e898b is 50, key is test_row_0/B:col10/1732619829404/Put/seqid=0 2024-11-26T11:17:10,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619830591, max timestamp=1732619830605Num uniques 1018; 2024-11-26T11:17:10,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:17:10,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619830591, max timestamp=1732619830605Num uniques 1018; 2024-11-26T11:17:10,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732619830591, max timestamp=1732619830605Num uniques 1018; 2024-11-26T11:17:10,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:17:10,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T11:17:10,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742334_1510 (size=12001) 2024-11-26T11:17:10,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/409d2818fd174d6ea2f8bb53ad3e898b 2024-11-26T11:17:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/0498c39621094c9baf278d705a51d0f1 is 50, key is test_row_0/C:col10/1732619829404/Put/seqid=0 2024-11-26T11:17:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619830624, max timestamp=1732619830653Num uniques 1077; 2024-11-26T11:17:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T11:17:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619830624, max timestamp=1732619830653Num uniques 1077; 2024-11-26T11:17:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732619830624, max timestamp=1732619830653Num uniques 1077; 2024-11-26T11:17:10,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T11:17:10,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T11:17:10,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742335_1511 (size=12001) 2024-11-26T11:17:10,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/0498c39621094c9baf278d705a51d0f1 2024-11-26T11:17:10,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d8f15cbe89a54b7bab5aa69d8943c17c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c 2024-11-26T11:17:10,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c, entries=150, sequenceid=70, filesize=30.2 K 2024-11-26T11:17:10,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/409d2818fd174d6ea2f8bb53ad3e898b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/409d2818fd174d6ea2f8bb53ad3e898b 2024-11-26T11:17:10,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/409d2818fd174d6ea2f8bb53ad3e898b, entries=150, sequenceid=70, filesize=11.7 K 2024-11-26T11:17:10,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/0498c39621094c9baf278d705a51d0f1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/0498c39621094c9baf278d705a51d0f1 2024-11-26T11:17:10,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/0498c39621094c9baf278d705a51d0f1, entries=150, sequenceid=70, filesize=11.7 K 2024-11-26T11:17:10,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~84.42 KB/86448, currentSize=134.18 KB/137400 for fa0c0e175d4b592132f0942e2a244b29 in 258ms, sequenceid=70, compaction requested=false 2024-11-26T11:17:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:10,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=147.60 KB heapSize=419.16 KB 2024-11-26T11:17:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T11:17:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T11:17:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T11:17:10,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c606f8d8b68a4bb7914460db24954ef2_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:10,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619890722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619890726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619890727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619890730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619890737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742336_1512 (size=19774) 2024-11-26T11:17:10,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619890842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619890843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619890853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619890853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619890853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:10,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:10,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:10,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:10,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619891053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619891053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619891063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619891063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619891063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,170 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:11,173 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c606f8d8b68a4bb7914460db24954ef2_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c606f8d8b68a4bb7914460db24954ef2_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:11,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/22ebd063f0f34e66a93881f7fa3b54c6, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:11,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/22ebd063f0f34e66a93881f7fa3b54c6 is 175, key is test_row_0/A:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:11,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742337_1513 (size=57033) 2024-11-26T11:17:11,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/22ebd063f0f34e66a93881f7fa3b54c6 2024-11-26T11:17:11,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/0827f33f6bf443ebb4eeb5c455736d62 is 50, key is test_row_0/B:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:11,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742338_1514 (size=12151) 2024-11-26T11:17:11,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/0827f33f6bf443ebb4eeb5c455736d62 2024-11-26T11:17:11,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/499d6d42961c445aa43b6125db229bd2 is 50, key is test_row_0/C:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:11,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742339_1515 (size=12151) 2024-11-26T11:17:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/499d6d42961c445aa43b6125db229bd2 2024-11-26T11:17:11,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/22ebd063f0f34e66a93881f7fa3b54c6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6 2024-11-26T11:17:11,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6, entries=300, sequenceid=143, filesize=55.7 K 2024-11-26T11:17:11,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/0827f33f6bf443ebb4eeb5c455736d62 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0827f33f6bf443ebb4eeb5c455736d62 2024-11-26T11:17:11,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0827f33f6bf443ebb4eeb5c455736d62, entries=150, sequenceid=143, filesize=11.9 K 2024-11-26T11:17:11,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/499d6d42961c445aa43b6125db229bd2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/499d6d42961c445aa43b6125db229bd2 2024-11-26T11:17:11,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/499d6d42961c445aa43b6125db229bd2, entries=150, sequenceid=143, filesize=11.9 K 2024-11-26T11:17:11,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~439.03 KB/449568, currentSize=33.54 KB/34350 for fa0c0e175d4b592132f0942e2a244b29 in 570ms, sequenceid=143, compaction requested=true 2024-11-26T11:17:11,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:11,279 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:11,280 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:11,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:11,281 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:11,281 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:11,281 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:11,281 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:11,281 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,281 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,281 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/53b1954aec65410e8abae7b43cf38570, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/409d2818fd174d6ea2f8bb53ad3e898b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0827f33f6bf443ebb4eeb5c455736d62] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=35.3 K 2024-11-26T11:17:11,281 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=116.2 K 2024-11-26T11:17:11,282 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,282 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6] 2024-11-26T11:17:11,282 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b1954aec65410e8abae7b43cf38570, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619829203 2024-11-26T11:17:11,282 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6a9e9c52b64387a9fc2300ebc94194, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619829203 2024-11-26T11:17:11,282 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 409d2818fd174d6ea2f8bb53ad3e898b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732619829404 2024-11-26T11:17:11,282 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f15cbe89a54b7bab5aa69d8943c17c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732619829404 2024-11-26T11:17:11,282 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0827f33f6bf443ebb4eeb5c455736d62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619830694 2024-11-26T11:17:11,283 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ebd063f0f34e66a93881f7fa3b54c6, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619830679 2024-11-26T11:17:11,293 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#435 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:11,293 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/239d72ad3bbd42feb367acf36c4a4826 is 50, key is test_row_0/B:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:11,305 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:11,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742340_1516 (size=12254) 2024-11-26T11:17:11,329 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411265e290226dda74beabb62cc5fbcab6533_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:11,332 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411265e290226dda74beabb62cc5fbcab6533_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:11,332 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265e290226dda74beabb62cc5fbcab6533_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:11,336 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/239d72ad3bbd42feb367acf36c4a4826 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/239d72ad3bbd42feb367acf36c4a4826 2024-11-26T11:17:11,343 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 239d72ad3bbd42feb367acf36c4a4826(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:11,343 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:11,343 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=13, startTime=1732619831280; duration=0sec 2024-11-26T11:17:11,343 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:11,343 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:11,343 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:11,345 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:11,345 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:11,345 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,346 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/32388a2d00324c56ada47cc15c7b6374, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/0498c39621094c9baf278d705a51d0f1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/499d6d42961c445aa43b6125db229bd2] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=35.3 K 2024-11-26T11:17:11,346 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 32388a2d00324c56ada47cc15c7b6374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732619829203 2024-11-26T11:17:11,346 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0498c39621094c9baf278d705a51d0f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732619829404 2024-11-26T11:17:11,347 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 499d6d42961c445aa43b6125db229bd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619830694 2024-11-26T11:17:11,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742341_1517 (size=4469) 2024-11-26T11:17:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:11,365 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:11,365 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3 is 50, key is test_row_0/C:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:11,366 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#436 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:11,367 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/45beab274ec24392a6507efe73bb9394 is 175, key is test_row_0/A:col10/1732619830694/Put/seqid=0 2024-11-26T11:17:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:11,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:17:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:11,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742342_1518 (size=31208) 2024-11-26T11:17:11,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e7e4e16031ef4fa98aab2cdec1f21334_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619831369/Put/seqid=0 2024-11-26T11:17:11,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742343_1519 (size=12254) 2024-11-26T11:17:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742344_1520 (size=14794) 2024-11-26T11:17:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619891396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619891399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732619831388, max timestamp=1732619831410Num uniques 859; 2024-11-26T11:17:11,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:11,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732619831388, max timestamp=1732619831410Num uniques 859; 2024-11-26T11:17:11,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:11,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732619831388, max timestamp=1732619831410Num uniques 859; 2024-11-26T11:17:11,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T11:17:11,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619891505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619891510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619891514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619891563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619891563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619891622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619891622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619891633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:11,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:11,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619891772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619891773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,813 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/45beab274ec24392a6507efe73bb9394 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/45beab274ec24392a6507efe73bb9394 2024-11-26T11:17:11,820 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:11,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e7e4e16031ef4fa98aab2cdec1f21334_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e7e4e16031ef4fa98aab2cdec1f21334_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:11,830 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/43fdf7a2909a4c3db12b69001d68b8ce, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:11,830 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into 45beab274ec24392a6507efe73bb9394(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:11,830 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:11,830 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=13, startTime=1732619831279; duration=0sec 2024-11-26T11:17:11,831 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:11,831 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:11,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/43fdf7a2909a4c3db12b69001d68b8ce is 175, key is test_row_0/A:col10/1732619831369/Put/seqid=0 2024-11-26T11:17:11,835 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3 2024-11-26T11:17:11,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742345_1521 (size=39749) 2024-11-26T11:17:11,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/43fdf7a2909a4c3db12b69001d68b8ce 2024-11-26T11:17:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619891833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619891833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619891843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:11,857 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into d4e61cbf21cd4f8592fe67cfb1dc9bf3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:11,857 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:11,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:11,857 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=13, startTime=1732619831280; duration=0sec 2024-11-26T11:17:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,857 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:11,857 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:11,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:11,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/1e1716b290fe431e8e134e66dc13b47f is 50, key is test_row_0/B:col10/1732619831369/Put/seqid=0 2024-11-26T11:17:11,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742346_1522 (size=12151) 2024-11-26T11:17:11,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/1e1716b290fe431e8e134e66dc13b47f 2024-11-26T11:17:11,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/b18f6b0dbb0945aea0f2618a15553df3 is 50, key is test_row_0/C:col10/1732619831369/Put/seqid=0 2024-11-26T11:17:11,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742347_1523 (size=12151) 2024-11-26T11:17:11,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/b18f6b0dbb0945aea0f2618a15553df3 2024-11-26T11:17:11,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/43fdf7a2909a4c3db12b69001d68b8ce as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce 2024-11-26T11:17:11,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce, entries=200, sequenceid=154, filesize=38.8 K 2024-11-26T11:17:11,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/1e1716b290fe431e8e134e66dc13b47f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1e1716b290fe431e8e134e66dc13b47f 2024-11-26T11:17:11,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1e1716b290fe431e8e134e66dc13b47f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-26T11:17:11,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/b18f6b0dbb0945aea0f2618a15553df3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b18f6b0dbb0945aea0f2618a15553df3 2024-11-26T11:17:11,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b18f6b0dbb0945aea0f2618a15553df3, entries=150, sequenceid=154, filesize=11.9 K 2024-11-26T11:17:11,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.56 KB/134716 for fa0c0e175d4b592132f0942e2a244b29 in 609ms, sequenceid=154, compaction requested=false 2024-11-26T11:17:11,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:12,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T11:17:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:12,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=131.56 KB heapSize=374.30 KB 2024-11-26T11:17:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:12,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126dbda699a3940402f8b3acc0a1573fbf3_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619831510/Put/seqid=0 2024-11-26T11:17:12,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742348_1524 (size=12304) 2024-11-26T11:17:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:12,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:12,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619892108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619892114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619892143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619892143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619892153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619892223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619892223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619892432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619892432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:12,458 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126dbda699a3940402f8b3acc0a1573fbf3_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dbda699a3940402f8b3acc0a1573fbf3_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:12,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/343566814b284001ab03e8c357be5ae3, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:12,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/343566814b284001ab03e8c357be5ae3 is 175, key is test_row_0/A:col10/1732619831510/Put/seqid=0 2024-11-26T11:17:12,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742349_1525 (size=31105) 2024-11-26T11:17:12,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619892652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619892653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619892663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619892743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619892743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:12,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/343566814b284001ab03e8c357be5ae3 2024-11-26T11:17:12,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/c758c03342084b1ba69b6e1f9254e259 is 50, key is test_row_0/B:col10/1732619831510/Put/seqid=0 2024-11-26T11:17:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742350_1526 (size=12151) 2024-11-26T11:17:12,911 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/c758c03342084b1ba69b6e1f9254e259 2024-11-26T11:17:12,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/758c94bbfabf4281beedfbc8ba5f69c1 is 50, key is test_row_0/C:col10/1732619831510/Put/seqid=0 2024-11-26T11:17:12,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742351_1527 (size=12151) 2024-11-26T11:17:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619893253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619893253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,338 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.11 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/758c94bbfabf4281beedfbc8ba5f69c1 2024-11-26T11:17:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/343566814b284001ab03e8c357be5ae3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3 2024-11-26T11:17:13,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3, entries=150, sequenceid=192, filesize=30.4 K 2024-11-26T11:17:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/c758c03342084b1ba69b6e1f9254e259 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/c758c03342084b1ba69b6e1f9254e259 2024-11-26T11:17:13,357 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/c758c03342084b1ba69b6e1f9254e259, entries=150, sequenceid=192, filesize=11.9 K 2024-11-26T11:17:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/758c94bbfabf4281beedfbc8ba5f69c1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/758c94bbfabf4281beedfbc8ba5f69c1 2024-11-26T11:17:13,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/758c94bbfabf4281beedfbc8ba5f69c1, entries=150, sequenceid=192, filesize=11.9 K 2024-11-26T11:17:13,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~131.56 KB/134716, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for fa0c0e175d4b592132f0942e2a244b29 in 1353ms, sequenceid=192, compaction requested=true 2024-11-26T11:17:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-26T11:17:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-26T11:17:13,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-26T11:17:13,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1440 sec 2024-11-26T11:17:13,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 4.1530 sec 2024-11-26T11:17:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T11:17:13,383 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:13,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-26T11:17:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T11:17:13,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:13,387 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:13,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T11:17:13,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-26T11:17:13,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:13,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:13,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ba69165a24ed4010a5f592bf4a195bbd_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619832094/Put/seqid=0 2024-11-26T11:17:13,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742352_1528 (size=12304) 2024-11-26T11:17:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:13,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T11:17:13,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619833698, max timestamp=1732619833728Num uniques 859; 2024-11-26T11:17:13,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:13,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619833698, max timestamp=1732619833728Num uniques 859; 2024-11-26T11:17:13,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619833698, max timestamp=1732619833728Num uniques 859; 2024-11-26T11:17:13,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:13,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619893772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619893779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619893781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619893893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619893893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619893893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:14,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ba69165a24ed4010a5f592bf4a195bbd_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ba69165a24ed4010a5f592bf4a195bbd_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:14,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/c51cd9695f3643ceb0b99e8d82815ad9, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:14,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/c51cd9695f3643ceb0b99e8d82815ad9 is 175, key is test_row_0/A:col10/1732619832094/Put/seqid=0 2024-11-26T11:17:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T11:17:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742353_1529 (size=31105) 2024-11-26T11:17:14,015 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/c51cd9695f3643ceb0b99e8d82815ad9 2024-11-26T11:17:14,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/4583de94b37843e2b305e700ae083cef is 50, key is test_row_0/B:col10/1732619832094/Put/seqid=0 2024-11-26T11:17:14,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742354_1530 (size=12151) 2024-11-26T11:17:14,057 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/4583de94b37843e2b305e700ae083cef 2024-11-26T11:17:14,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/514f35521f3e4273b073fccdefe54b5a is 50, key is test_row_0/C:col10/1732619832094/Put/seqid=0 2024-11-26T11:17:14,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742355_1531 (size=12151) 2024-11-26T11:17:14,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/514f35521f3e4273b073fccdefe54b5a 2024-11-26T11:17:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/c51cd9695f3643ceb0b99e8d82815ad9 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9 2024-11-26T11:17:14,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619894113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619894113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619894114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,122 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9, entries=150, sequenceid=202, filesize=30.4 K 2024-11-26T11:17:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/4583de94b37843e2b305e700ae083cef as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/4583de94b37843e2b305e700ae083cef 2024-11-26T11:17:14,128 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/4583de94b37843e2b305e700ae083cef, entries=150, sequenceid=202, filesize=11.9 K 2024-11-26T11:17:14,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/514f35521f3e4273b073fccdefe54b5a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/514f35521f3e4273b073fccdefe54b5a 2024-11-26T11:17:14,134 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/514f35521f3e4273b073fccdefe54b5a, entries=150, sequenceid=202, filesize=11.9 K 2024-11-26T11:17:14,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fa0c0e175d4b592132f0942e2a244b29 in 594ms, sequenceid=202, compaction requested=true 2024-11-26T11:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-26T11:17:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-26T11:17:14,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-26T11:17:14,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-26T11:17:14,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 757 msec 2024-11-26T11:17:14,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T11:17:14,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:14,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:14,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:14,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:14,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:14,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:14,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112617bc2e54fd4f4a858828472bfe012c86_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:14,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742356_1532 (size=14794) 2024-11-26T11:17:14,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:14,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112617bc2e54fd4f4a858828472bfe012c86_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112617bc2e54fd4f4a858828472bfe012c86_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:14,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/51588c7f375d400ea13a5d8db74ae87e, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:14,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619894282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/51588c7f375d400ea13a5d8db74ae87e is 175, key is test_row_0/A:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:14,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742357_1533 (size=39749) 2024-11-26T11:17:14,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619894287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619894396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619894403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619894423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619894423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619894423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,487 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T11:17:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T11:17:14,523 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:14,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-26T11:17:14,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:14,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:14,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T11:17:14,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619894602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619894613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T11:17:14,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-26T11:17:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/51588c7f375d400ea13a5d8db74ae87e 2024-11-26T11:17:14,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/0cdcd7b2e0624932b286a7f4c14be3cc is 50, key is test_row_0/B:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:14,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742358_1534 (size=12151) 2024-11-26T11:17:14,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-26T11:17:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T11:17:14,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619894913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619894923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619894933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619894932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619894943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:14,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-26T11:17:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:14,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/0cdcd7b2e0624932b286a7f4c14be3cc 2024-11-26T11:17:15,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/2c3c8d9e3bf54c51a4f7c319d9e92e38 is 50, key is test_row_0/C:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:15,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742359_1535 (size=12151) 2024-11-26T11:17:15,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/2c3c8d9e3bf54c51a4f7c319d9e92e38 2024-11-26T11:17:15,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-26T11:17:15,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:15,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/51588c7f375d400ea13a5d8db74ae87e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e 2024-11-26T11:17:15,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e, entries=200, sequenceid=239, filesize=38.8 K 2024-11-26T11:17:15,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/0cdcd7b2e0624932b286a7f4c14be3cc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0cdcd7b2e0624932b286a7f4c14be3cc 2024-11-26T11:17:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0cdcd7b2e0624932b286a7f4c14be3cc, entries=150, sequenceid=239, filesize=11.9 K 2024-11-26T11:17:15,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/2c3c8d9e3bf54c51a4f7c319d9e92e38 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/2c3c8d9e3bf54c51a4f7c319d9e92e38 2024-11-26T11:17:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/2c3c8d9e3bf54c51a4f7c319d9e92e38, entries=150, sequenceid=239, filesize=11.9 K 2024-11-26T11:17:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fa0c0e175d4b592132f0942e2a244b29 in 897ms, sequenceid=239, compaction requested=true 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:15,163 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:15,163 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T11:17:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,164 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60858 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T11:17:15,165 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:15,165 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,165 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/239d72ad3bbd42feb367acf36c4a4826, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1e1716b290fe431e8e134e66dc13b47f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/c758c03342084b1ba69b6e1f9254e259, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/4583de94b37843e2b305e700ae083cef, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0cdcd7b2e0624932b286a7f4c14be3cc] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=59.4 K 2024-11-26T11:17:15,165 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172916 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T11:17:15,165 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:15,165 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,165 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/45beab274ec24392a6507efe73bb9394, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=168.9 K 2024-11-26T11:17:15,165 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 239d72ad3bbd42feb367acf36c4a4826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619830694 2024-11-26T11:17:15,165 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,165 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/45beab274ec24392a6507efe73bb9394, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e] 2024-11-26T11:17:15,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1716b290fe431e8e134e66dc13b47f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732619830726 2024-11-26T11:17:15,166 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45beab274ec24392a6507efe73bb9394, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619830694 2024-11-26T11:17:15,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c758c03342084b1ba69b6e1f9254e259, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732619831477 2024-11-26T11:17:15,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 4583de94b37843e2b305e700ae083cef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732619832094 2024-11-26T11:17:15,166 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cdcd7b2e0624932b286a7f4c14be3cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732619833767 2024-11-26T11:17:15,167 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43fdf7a2909a4c3db12b69001d68b8ce, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732619830723 2024-11-26T11:17:15,167 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343566814b284001ab03e8c357be5ae3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732619831477 2024-11-26T11:17:15,167 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51cd9695f3643ceb0b99e8d82815ad9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732619832094 2024-11-26T11:17:15,168 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51588c7f375d400ea13a5d8db74ae87e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732619833767 2024-11-26T11:17:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,175 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,176 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/137cd646406b42c28d1681b0f7fe436c is 50, key is test_row_0/B:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:15,176 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,180 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112650360dbd35d3472e87c74c3fa1d25679_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:15,183 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112650360dbd35d3472e87c74c3fa1d25679_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:15,183 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112650360dbd35d3472e87c74c3fa1d25679_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:15,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742360_1536 (size=12425) 2024-11-26T11:17:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742361_1537 (size=4469) 2024-11-26T11:17:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,199 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/137cd646406b42c28d1681b0f7fe436c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/137cd646406b42c28d1681b0f7fe436c 2024-11-26T11:17:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,206 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 137cd646406b42c28d1681b0f7fe436c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:15,206 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:15,206 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=11, startTime=1732619835163; duration=0sec 2024-11-26T11:17:15,206 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:15,206 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:15,206 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T11:17:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,209 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60858 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T11:17:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,209 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:15,209 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,209 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b18f6b0dbb0945aea0f2618a15553df3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/758c94bbfabf4281beedfbc8ba5f69c1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/514f35521f3e4273b073fccdefe54b5a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/2c3c8d9e3bf54c51a4f7c319d9e92e38] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=59.4 K 2024-11-26T11:17:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,209 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e61cbf21cd4f8592fe67cfb1dc9bf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732619830694 2024-11-26T11:17:15,210 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b18f6b0dbb0945aea0f2618a15553df3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732619830726 2024-11-26T11:17:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,210 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 758c94bbfabf4281beedfbc8ba5f69c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732619831477 2024-11-26T11:17:15,210 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 514f35521f3e4273b073fccdefe54b5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732619832094 2024-11-26T11:17:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,210 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3c8d9e3bf54c51a4f7c319d9e92e38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732619833767 2024-11-26T11:17:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,227 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:15,228 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/295f48eb9a034f1889a98e45669689a2 is 50, key is test_row_0/C:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742362_1538 (size=12425) 2024-11-26T11:17:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,286 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/295f48eb9a034f1889a98e45669689a2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/295f48eb9a034f1889a98e45669689a2 2024-11-26T11:17:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,292 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into 295f48eb9a034f1889a98e45669689a2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:15,292 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:15,292 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=11, startTime=1732619835163; duration=0sec 2024-11-26T11:17:15,292 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:15,292 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-26T11:17:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,304 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T11:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e8030af222d4481aec5bec62811ccc4_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619834280/Put/seqid=0 2024-11-26T11:17:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742363_1539 (size=9814) 2024-11-26T11:17:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e8030af222d4481aec5bec62811ccc4_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e8030af222d4481aec5bec62811ccc4_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/11476c431b484addb3bb9eb8709a0160, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/11476c431b484addb3bb9eb8709a0160 is 175, key is test_row_0/A:col10/1732619834280/Put/seqid=0 2024-11-26T11:17:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742364_1540 (size=22461) 2024-11-26T11:17:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/11476c431b484addb3bb9eb8709a0160 2024-11-26T11:17:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/1789c6199d4941679e2aa2430bb8b58e is 50, key is test_row_0/B:col10/1732619834280/Put/seqid=0 2024-11-26T11:17:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742365_1541 (size=9757) 2024-11-26T11:17:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,415 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/1789c6199d4941679e2aa2430bb8b58e 2024-11-26T11:17:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/33dc7e61eb004b428f60302c55579172 is 50, key is test_row_0/C:col10/1732619834280/Put/seqid=0 2024-11-26T11:17:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742366_1542 (size=9757) 2024-11-26T11:17:15,430 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/33dc7e61eb004b428f60302c55579172 2024-11-26T11:17:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/11476c431b484addb3bb9eb8709a0160 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160 2024-11-26T11:17:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,445 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160, entries=100, sequenceid=250, filesize=21.9 K 2024-11-26T11:17:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/1789c6199d4941679e2aa2430bb8b58e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1789c6199d4941679e2aa2430bb8b58e 2024-11-26T11:17:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,449 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1789c6199d4941679e2aa2430bb8b58e, entries=100, sequenceid=250, filesize=9.5 K 2024-11-26T11:17:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/33dc7e61eb004b428f60302c55579172 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/33dc7e61eb004b428f60302c55579172 2024-11-26T11:17:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/33dc7e61eb004b428f60302c55579172, entries=100, sequenceid=250, filesize=9.5 K 2024-11-26T11:17:15,454 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=20.13 KB/20610 for fa0c0e175d4b592132f0942e2a244b29 in 150ms, sequenceid=250, compaction requested=false 2024-11-26T11:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-26T11:17:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-26T11:17:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-26T11:17:15,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 928 msec 2024-11-26T11:17:15,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 935 msec 2024-11-26T11:17:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:17:15,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:15,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:15,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:15,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:15,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T11:17:15,482 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T11:17:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112678bf00c9917a4668bd0ff5cb8a9f0ba2_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742367_1543 (size=19974) 2024-11-26T11:17:15,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619835532, max timestamp=1732619835554Num uniques 859; 2024-11-26T11:17:15,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619835532, max timestamp=1732619835554Num uniques 859; 2024-11-26T11:17:15,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:15,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732619835532, max timestamp=1732619835554Num uniques 859; 2024-11-26T11:17:15,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:15,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T11:17:15,600 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#451 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:15,601 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/48b4fae6dac8497fa9655da7d41f53f2 is 175, key is test_row_0/A:col10/1732619834265/Put/seqid=0 2024-11-26T11:17:15,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742368_1544 (size=31379) 2024-11-26T11:17:15,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619895613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619895624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T11:17:15,673 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:15,674 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-26T11:17:15,677 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:15,679 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:15,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:15,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619895732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619895743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:15,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T11:17:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,932 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:15,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112678bf00c9917a4668bd0ff5cb8a9f0ba2_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112678bf00c9917a4668bd0ff5cb8a9f0ba2_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:15,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/bd8c89894b484f7bbe6c9af5ae9021f0, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:15,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/bd8c89894b484f7bbe6c9af5ae9021f0 is 175, key is test_row_0/A:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:15,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742369_1545 (size=57233) 2024-11-26T11:17:15,947 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/bd8c89894b484f7bbe6c9af5ae9021f0 2024-11-26T11:17:15,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619895942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/08d18551510b4d7490aa94b902d483c3 is 50, key is test_row_0/B:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619895954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619895954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,959 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619895957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,962 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619895957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:15,964 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:15,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742370_1546 (size=12301) 2024-11-26T11:17:15,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T11:17:15,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:15,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:16,014 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/48b4fae6dac8497fa9655da7d41f53f2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/48b4fae6dac8497fa9655da7d41f53f2 2024-11-26T11:17:16,020 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into 48b4fae6dac8497fa9655da7d41f53f2(size=30.6 K), total size for store is 52.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:16,020 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:16,020 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=11, startTime=1732619835163; duration=0sec 2024-11-26T11:17:16,020 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:16,020 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:16,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T11:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619896253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619896263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:16,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T11:17:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/08d18551510b4d7490aa94b902d483c3 2024-11-26T11:17:16,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/66127a572d074637bee730c08e08af1d is 50, key is test_row_0/C:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:16,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T11:17:16,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:16,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:16,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742371_1547 (size=12301) 2024-11-26T11:17:16,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/66127a572d074637bee730c08e08af1d 2024-11-26T11:17:16,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/bd8c89894b484f7bbe6c9af5ae9021f0 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0 2024-11-26T11:17:16,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0, entries=300, sequenceid=260, filesize=55.9 K 2024-11-26T11:17:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/08d18551510b4d7490aa94b902d483c3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/08d18551510b4d7490aa94b902d483c3 2024-11-26T11:17:16,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/08d18551510b4d7490aa94b902d483c3, entries=150, sequenceid=260, filesize=12.0 K 2024-11-26T11:17:16,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/66127a572d074637bee730c08e08af1d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/66127a572d074637bee730c08e08af1d 2024-11-26T11:17:16,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/66127a572d074637bee730c08e08af1d, entries=150, sequenceid=260, filesize=12.0 K 2024-11-26T11:17:16,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fa0c0e175d4b592132f0942e2a244b29 in 1042ms, sequenceid=260, compaction requested=true 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:16,512 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:16,512 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:16,513 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:16,513 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:16,513 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,513 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/48b4fae6dac8497fa9655da7d41f53f2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=108.5 K 2024-11-26T11:17:16,513 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,513 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/48b4fae6dac8497fa9655da7d41f53f2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0] 2024-11-26T11:17:16,513 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:16,514 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:16,514 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,514 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/137cd646406b42c28d1681b0f7fe436c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1789c6199d4941679e2aa2430bb8b58e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/08d18551510b4d7490aa94b902d483c3] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=33.7 K 2024-11-26T11:17:16,514 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b4fae6dac8497fa9655da7d41f53f2, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732619833767 2024-11-26T11:17:16,514 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 137cd646406b42c28d1681b0f7fe436c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732619833767 2024-11-26T11:17:16,514 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11476c431b484addb3bb9eb8709a0160, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732619834280 2024-11-26T11:17:16,514 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1789c6199d4941679e2aa2430bb8b58e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732619834280 2024-11-26T11:17:16,514 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd8c89894b484f7bbe6c9af5ae9021f0, keycount=300, bloomtype=ROW, size=55.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732619835433 2024-11-26T11:17:16,515 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d18551510b4d7490aa94b902d483c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732619835455 2024-11-26T11:17:16,529 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:16,538 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:16,539 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/079842749cff42bea7b344bdff8d4151 is 50, key is test_row_0/B:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:16,549 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112637518fd9b669464780fed8f6f94af6b7_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:16,551 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112637518fd9b669464780fed8f6f94af6b7_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:16,551 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112637518fd9b669464780fed8f6f94af6b7_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742372_1548 (size=12677) 2024-11-26T11:17:16,608 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/079842749cff42bea7b344bdff8d4151 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/079842749cff42bea7b344bdff8d4151 2024-11-26T11:17:16,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T11:17:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,611 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T11:17:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:16,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:16,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742373_1549 (size=4469) 2024-11-26T11:17:16,618 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 079842749cff42bea7b344bdff8d4151(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:16,618 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:16,618 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=13, startTime=1732619836512; duration=0sec 2024-11-26T11:17:16,618 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:16,618 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:16,619 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:16,621 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:16,621 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:16,622 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:16,622 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/295f48eb9a034f1889a98e45669689a2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/33dc7e61eb004b428f60302c55579172, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/66127a572d074637bee730c08e08af1d] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=33.7 K 2024-11-26T11:17:16,622 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 295f48eb9a034f1889a98e45669689a2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732619833767 2024-11-26T11:17:16,622 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 33dc7e61eb004b428f60302c55579172, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732619834280 2024-11-26T11:17:16,623 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 66127a572d074637bee730c08e08af1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732619835455 2024-11-26T11:17:16,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f38e6d87379242ca8ba14da344ee678d_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619835621/Put/seqid=0 2024-11-26T11:17:16,642 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:16,643 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/dec5974467d54abc884a67bcec2c5337 is 50, key is test_row_0/C:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:16,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742374_1550 (size=12454) 2024-11-26T11:17:16,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742375_1551 (size=12677) 2024-11-26T11:17:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619896794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:16,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619896800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:16,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619896903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:16,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619896913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,015 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#459 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:17,015 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/f92fd9ccb0e0434db8d7f46b03cf1e06 is 175, key is test_row_0/A:col10/1732619835455/Put/seqid=0 2024-11-26T11:17:17,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742376_1552 (size=31631) 2024-11-26T11:17:17,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:17,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f38e6d87379242ca8ba14da344ee678d_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f38e6d87379242ca8ba14da344ee678d_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:17,094 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/dec5974467d54abc884a67bcec2c5337 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/dec5974467d54abc884a67bcec2c5337 2024-11-26T11:17:17,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8b202f9aed134631b34db39066735a4e, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8b202f9aed134631b34db39066735a4e is 175, key is test_row_0/A:col10/1732619835621/Put/seqid=0 2024-11-26T11:17:17,100 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into dec5974467d54abc884a67bcec2c5337(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:17,100 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:17,100 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=13, startTime=1732619836512; duration=0sec 2024-11-26T11:17:17,100 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:17,100 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:17,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619897113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742377_1553 (size=31255) 2024-11-26T11:17:17,123 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8b202f9aed134631b34db39066735a4e 2024-11-26T11:17:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619897123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/af6e905867ef42d88a94c8a9986c7e9d is 50, key is test_row_0/B:col10/1732619835621/Put/seqid=0 2024-11-26T11:17:17,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742378_1554 (size=12301) 2024-11-26T11:17:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619897423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,433 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/f92fd9ccb0e0434db8d7f46b03cf1e06 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f92fd9ccb0e0434db8d7f46b03cf1e06 2024-11-26T11:17:17,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,436 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into f92fd9ccb0e0434db8d7f46b03cf1e06(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:17,436 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619897433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,436 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=13, startTime=1732619836512; duration=0sec 2024-11-26T11:17:17,436 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:17,436 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,542 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/af6e905867ef42d88a94c8a9986c7e9d 2024-11-26T11:17:17,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/70b9733e27884bd8911bfc82dc6ebe62 is 50, key is test_row_0/C:col10/1732619835621/Put/seqid=0 2024-11-26T11:17:17,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742379_1555 (size=12301) 2024-11-26T11:17:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:17,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619897933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619897943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/70b9733e27884bd8911bfc82dc6ebe62 2024-11-26T11:17:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8b202f9aed134631b34db39066735a4e as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e 2024-11-26T11:17:17,969 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e, entries=150, sequenceid=297, filesize=30.5 K 2024-11-26T11:17:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/af6e905867ef42d88a94c8a9986c7e9d as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/af6e905867ef42d88a94c8a9986c7e9d 2024-11-26T11:17:17,976 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/af6e905867ef42d88a94c8a9986c7e9d, entries=150, sequenceid=297, filesize=12.0 K 2024-11-26T11:17:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/70b9733e27884bd8911bfc82dc6ebe62 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/70b9733e27884bd8911bfc82dc6ebe62 2024-11-26T11:17:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619897972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619897972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:17,981 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,982 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:17,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:17,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:17,984 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/70b9733e27884bd8911bfc82dc6ebe62, entries=150, sequenceid=297, filesize=12.0 K 2024-11-26T11:17:17,985 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for fa0c0e175d4b592132f0942e2a244b29 in 1375ms, sequenceid=297, compaction requested=false 2024-11-26T11:17:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-26T11:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-26T11:17:17,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T11:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:17,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-26T11:17:17,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3080 sec 2024-11-26T11:17:17,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.3230 sec 2024-11-26T11:17:18,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112698600fd7af09479bb55ad06859134ec9_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742380_1556 (size=12454) 2024-11-26T11:17:18,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:18,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112698600fd7af09479bb55ad06859134ec9_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112698600fd7af09479bb55ad06859134ec9_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:18,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/0df1f3c87be24c6390fc5b58623a76be, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:18,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/0df1f3c87be24c6390fc5b58623a76be is 175, key is test_row_0/A:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742381_1557 (size=31255) 2024-11-26T11:17:18,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/0df1f3c87be24c6390fc5b58623a76be 2024-11-26T11:17:18,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/945d641ce9754e7e9502bbf7658211e1 is 50, key is test_row_0/B:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742382_1558 (size=12301) 2024-11-26T11:17:18,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619838075, max timestamp=1732619838124Num uniques 859; 2024-11-26T11:17:18,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619838075, max timestamp=1732619838124Num uniques 859; 2024-11-26T11:17:18,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:18,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:18,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732619838075, max timestamp=1732619838124Num uniques 859; 2024-11-26T11:17:18,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7f3a3edaecc3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T11:17:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619898217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:18,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619898333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:18,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/945d641ce9754e7e9502bbf7658211e1 2024-11-26T11:17:18,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/71a6877770b142699e446d1daad42dc2 is 50, key is test_row_0/C:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742383_1559 (size=12301) 2024-11-26T11:17:18,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/71a6877770b142699e446d1daad42dc2 2024-11-26T11:17:18,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/0df1f3c87be24c6390fc5b58623a76be as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be 2024-11-26T11:17:18,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be, entries=150, sequenceid=311, filesize=30.5 K 2024-11-26T11:17:18,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/945d641ce9754e7e9502bbf7658211e1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/945d641ce9754e7e9502bbf7658211e1 2024-11-26T11:17:18,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/945d641ce9754e7e9502bbf7658211e1, entries=150, sequenceid=311, filesize=12.0 K 2024-11-26T11:17:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/71a6877770b142699e446d1daad42dc2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/71a6877770b142699e446d1daad42dc2 2024-11-26T11:17:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619898542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:18,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/71a6877770b142699e446d1daad42dc2, entries=150, sequenceid=311, filesize=12.0 K 2024-11-26T11:17:18,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fa0c0e175d4b592132f0942e2a244b29 in 574ms, sequenceid=311, compaction requested=true 2024-11-26T11:17:18,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:18,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:18,564 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:18,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:18,565 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:18,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:18,566 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:18,566 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:18,566 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:18,567 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/079842749cff42bea7b344bdff8d4151, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/af6e905867ef42d88a94c8a9986c7e9d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/945d641ce9754e7e9502bbf7658211e1] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=36.4 K 2024-11-26T11:17:18,567 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:18,567 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:18,567 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:18,567 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f92fd9ccb0e0434db8d7f46b03cf1e06, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=91.9 K 2024-11-26T11:17:18,567 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 079842749cff42bea7b344bdff8d4151, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732619835455 2024-11-26T11:17:18,567 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:18,567 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f92fd9ccb0e0434db8d7f46b03cf1e06, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be] 2024-11-26T11:17:18,567 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting af6e905867ef42d88a94c8a9986c7e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732619835599 2024-11-26T11:17:18,567 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f92fd9ccb0e0434db8d7f46b03cf1e06, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732619835455 2024-11-26T11:17:18,568 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 945d641ce9754e7e9502bbf7658211e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732619836792 2024-11-26T11:17:18,568 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b202f9aed134631b34db39066735a4e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732619835599 2024-11-26T11:17:18,568 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df1f3c87be24c6390fc5b58623a76be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732619836792 2024-11-26T11:17:18,585 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:18,598 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:18,599 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/83813c9b862242448bbdfbaad2d60a43 is 50, key is test_row_0/B:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,609 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411265a2591c1726b4a34b23cb5de76409bc2_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:18,611 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411265a2591c1726b4a34b23cb5de76409bc2_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:18,611 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a2591c1726b4a34b23cb5de76409bc2_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:18,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742384_1560 (size=12779) 2024-11-26T11:17:18,638 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/83813c9b862242448bbdfbaad2d60a43 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/83813c9b862242448bbdfbaad2d60a43 2024-11-26T11:17:18,645 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 83813c9b862242448bbdfbaad2d60a43(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:18,645 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:18,645 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=13, startTime=1732619838565; duration=0sec 2024-11-26T11:17:18,645 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:18,645 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:18,645 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:18,646 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:18,646 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:18,646 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:18,647 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/dec5974467d54abc884a67bcec2c5337, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/70b9733e27884bd8911bfc82dc6ebe62, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/71a6877770b142699e446d1daad42dc2] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=36.4 K 2024-11-26T11:17:18,647 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting dec5974467d54abc884a67bcec2c5337, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732619835455 2024-11-26T11:17:18,648 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b9733e27884bd8911bfc82dc6ebe62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732619835599 2024-11-26T11:17:18,648 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a6877770b142699e446d1daad42dc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732619836792 2024-11-26T11:17:18,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742385_1561 (size=4469) 2024-11-26T11:17:18,656 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#468 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:18,656 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/bbb2cd4a0fe94efe85365bfe478d68c3 is 175, key is test_row_0/A:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,666 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:18,667 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/1348b9ed87484f7293038c6acb98ff26 is 50, key is test_row_0/C:col10/1732619837988/Put/seqid=0 2024-11-26T11:17:18,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742386_1562 (size=31733) 2024-11-26T11:17:18,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742387_1563 (size=12779) 2024-11-26T11:17:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:18,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T11:17:18,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T11:17:18,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c90ff0a955ab4fd0b10eece7b36a8138_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619838151/Put/seqid=0 2024-11-26T11:17:18,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742388_1564 (size=12454) 2024-11-26T11:17:18,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:18,882 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c90ff0a955ab4fd0b10eece7b36a8138_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c90ff0a955ab4fd0b10eece7b36a8138_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:18,885 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/faa263ff1bb64a9fb71435c6fe9e49a1, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:18,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/faa263ff1bb64a9fb71435c6fe9e49a1 is 175, key is test_row_0/A:col10/1732619838151/Put/seqid=0 2024-11-26T11:17:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742389_1565 (size=31255) 2024-11-26T11:17:18,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/faa263ff1bb64a9fb71435c6fe9e49a1 2024-11-26T11:17:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619898902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:18,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/ab0484a4a49b435881da4e23f907f061 is 50, key is test_row_0/B:col10/1732619838151/Put/seqid=0 2024-11-26T11:17:18,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742390_1566 (size=12301) 2024-11-26T11:17:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619898943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:18,949 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:18,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619898963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:18,964 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:19,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619899013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:19,103 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/bbb2cd4a0fe94efe85365bfe478d68c3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bbb2cd4a0fe94efe85365bfe478d68c3 2024-11-26T11:17:19,108 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/1348b9ed87484f7293038c6acb98ff26 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1348b9ed87484f7293038c6acb98ff26 2024-11-26T11:17:19,108 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into bbb2cd4a0fe94efe85365bfe478d68c3(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:19,108 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:19,108 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=13, startTime=1732619838564; duration=0sec 2024-11-26T11:17:19,108 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:19,109 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:19,119 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into 1348b9ed87484f7293038c6acb98ff26(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:19,119 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:19,119 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=13, startTime=1732619838566; duration=0sec 2024-11-26T11:17:19,119 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:19,119 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619899224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:19,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/ab0484a4a49b435881da4e23f907f061 2024-11-26T11:17:19,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/21b0bf501a5f422d8ca24472f76b5031 is 50, key is test_row_0/C:col10/1732619838151/Put/seqid=0 2024-11-26T11:17:19,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742391_1567 (size=12301) 2024-11-26T11:17:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619899533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:19,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/21b0bf501a5f422d8ca24472f76b5031 2024-11-26T11:17:19,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/faa263ff1bb64a9fb71435c6fe9e49a1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1 2024-11-26T11:17:19,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1, entries=150, sequenceid=349, filesize=30.5 K 2024-11-26T11:17:19,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/ab0484a4a49b435881da4e23f907f061 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ab0484a4a49b435881da4e23f907f061 2024-11-26T11:17:19,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ab0484a4a49b435881da4e23f907f061, entries=150, sequenceid=349, filesize=12.0 K 2024-11-26T11:17:19,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/21b0bf501a5f422d8ca24472f76b5031 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/21b0bf501a5f422d8ca24472f76b5031 2024-11-26T11:17:19,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/21b0bf501a5f422d8ca24472f76b5031, entries=150, sequenceid=349, filesize=12.0 K 2024-11-26T11:17:19,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fa0c0e175d4b592132f0942e2a244b29 in 917ms, sequenceid=349, compaction requested=false 2024-11-26T11:17:19,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T11:17:19,853 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:19,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-26T11:17:19,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T11:17:19,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:19,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T11:17:20,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,010 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:20,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e282f1d22f3a44ab860a0ff60e1a5622_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619838899/Put/seqid=0 2024-11-26T11:17:20,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742392_1568 (size=12454) 2024-11-26T11:17:20,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:20,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e282f1d22f3a44ab860a0ff60e1a5622_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e282f1d22f3a44ab860a0ff60e1a5622_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:20,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/f5c413e7eb0c4ce787cdb5e5f04b65c3, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:20,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/f5c413e7eb0c4ce787cdb5e5f04b65c3 is 175, key is test_row_0/A:col10/1732619838899/Put/seqid=0 2024-11-26T11:17:20,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:20,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742393_1569 (size=31255) 2024-11-26T11:17:20,069 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/f5c413e7eb0c4ce787cdb5e5f04b65c3 2024-11-26T11:17:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/bbd3c3e783ea41139fa39225f00b55eb is 50, key is test_row_0/B:col10/1732619838899/Put/seqid=0 2024-11-26T11:17:20,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742394_1570 (size=12301) 2024-11-26T11:17:20,092 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/bbd3c3e783ea41139fa39225f00b55eb 2024-11-26T11:17:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/8f2954257fde433999930566bae7ffbf is 50, key is test_row_0/C:col10/1732619838899/Put/seqid=0 2024-11-26T11:17:20,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742395_1571 (size=12301) 2024-11-26T11:17:20,106 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/8f2954257fde433999930566bae7ffbf 2024-11-26T11:17:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/f5c413e7eb0c4ce787cdb5e5f04b65c3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3 2024-11-26T11:17:20,113 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3, entries=150, sequenceid=361, filesize=30.5 K 2024-11-26T11:17:20,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/bbd3c3e783ea41139fa39225f00b55eb as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/bbd3c3e783ea41139fa39225f00b55eb 2024-11-26T11:17:20,116 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/bbd3c3e783ea41139fa39225f00b55eb, entries=150, sequenceid=361, filesize=12.0 K 2024-11-26T11:17:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/8f2954257fde433999930566bae7ffbf as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/8f2954257fde433999930566bae7ffbf 2024-11-26T11:17:20,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/8f2954257fde433999930566bae7ffbf, entries=150, sequenceid=361, filesize=12.0 K 2024-11-26T11:17:20,121 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=87.22 KB/89310 for fa0c0e175d4b592132f0942e2a244b29 in 111ms, sequenceid=361, compaction requested=true 2024-11-26T11:17:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-26T11:17:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-26T11:17:20,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:17:20,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-26T11:17:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:20,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 267 msec 2024-11-26T11:17:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:20,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 272 msec 2024-11-26T11:17:20,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ff1bb2e82f044e0091d5b2b80b703aa1_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:20,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742396_1572 (size=14994) 2024-11-26T11:17:20,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:20,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ff1bb2e82f044e0091d5b2b80b703aa1_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ff1bb2e82f044e0091d5b2b80b703aa1_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:20,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/a3e2fed278b84bfe877266ed2a420610, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:20,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/a3e2fed278b84bfe877266ed2a420610 is 175, key is test_row_0/A:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:20,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742397_1573 (size=39949) 2024-11-26T11:17:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T11:17:20,173 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:20,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-26T11:17:20,175 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T11:17:20,176 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:20,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:20,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619900194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T11:17:20,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619900303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:20,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T11:17:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:20,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T11:17:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T11:17:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619900512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:20,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/a3e2fed278b84bfe877266ed2a420610 2024-11-26T11:17:20,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/b7cee3e571394c2482557adb67b7b6b6 is 50, key is test_row_0/B:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:20,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742398_1574 (size=12301) 2024-11-26T11:17:20,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/b7cee3e571394c2482557adb67b7b6b6 2024-11-26T11:17:20,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/3f580d91184e4c96960ce4130d49a9c2 is 50, key is test_row_0/C:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:20,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742399_1575 (size=12301) 2024-11-26T11:17:20,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T11:17:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T11:17:20,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:20,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T11:17:20,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619900822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:20,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T11:17:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:20,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:20,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:20,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619900973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:20,976 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619900983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:20,987 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:20,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/3f580d91184e4c96960ce4130d49a9c2 2024-11-26T11:17:20,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/a3e2fed278b84bfe877266ed2a420610 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610 2024-11-26T11:17:21,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610, entries=200, sequenceid=378, filesize=39.0 K 2024-11-26T11:17:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/b7cee3e571394c2482557adb67b7b6b6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/b7cee3e571394c2482557adb67b7b6b6 2024-11-26T11:17:21,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/b7cee3e571394c2482557adb67b7b6b6, entries=150, sequenceid=378, filesize=12.0 K 2024-11-26T11:17:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/3f580d91184e4c96960ce4130d49a9c2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/3f580d91184e4c96960ce4130d49a9c2 2024-11-26T11:17:21,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/3f580d91184e4c96960ce4130d49a9c2, entries=150, sequenceid=378, filesize=12.0 K 2024-11-26T11:17:21,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fa0c0e175d4b592132f0942e2a244b29 in 891ms, sequenceid=378, compaction requested=true 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:21,015 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:21,015 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:21,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:21,016 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:17:21,016 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:17:21,016 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:21,016 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:21,016 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:21,016 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:21,016 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bbb2cd4a0fe94efe85365bfe478d68c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=131.0 K 2024-11-26T11:17:21,016 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/83813c9b862242448bbdfbaad2d60a43, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ab0484a4a49b435881da4e23f907f061, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/bbd3c3e783ea41139fa39225f00b55eb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/b7cee3e571394c2482557adb67b7b6b6] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=48.5 K 2024-11-26T11:17:21,016 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:21,016 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bbb2cd4a0fe94efe85365bfe478d68c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610] 2024-11-26T11:17:21,017 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 83813c9b862242448bbdfbaad2d60a43, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732619836792 2024-11-26T11:17:21,017 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb2cd4a0fe94efe85365bfe478d68c3, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732619836792 2024-11-26T11:17:21,017 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0484a4a49b435881da4e23f907f061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732619838151 2024-11-26T11:17:21,018 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa263ff1bb64a9fb71435c6fe9e49a1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732619838151 2024-11-26T11:17:21,018 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd3c3e783ea41139fa39225f00b55eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732619838867 2024-11-26T11:17:21,018 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c413e7eb0c4ce787cdb5e5f04b65c3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732619838867 2024-11-26T11:17:21,018 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b7cee3e571394c2482557adb67b7b6b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732619840089 2024-11-26T11:17:21,018 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e2fed278b84bfe877266ed2a420610, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732619840089 2024-11-26T11:17:21,047 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:21,047 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/6d09695c4b7e486aab9d3610f92c5670 is 50, key is test_row_0/B:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:21,061 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:21,077 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112676096ff753c844a1aea0c526923ed821_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:21,080 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112676096ff753c844a1aea0c526923ed821_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:21,080 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112676096ff753c844a1aea0c526923ed821_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:21,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742400_1576 (size=12915) 2024-11-26T11:17:21,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T11:17:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:21,094 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T11:17:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:21,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:21,101 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/6d09695c4b7e486aab9d3610f92c5670 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/6d09695c4b7e486aab9d3610f92c5670 2024-11-26T11:17:21,107 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 6d09695c4b7e486aab9d3610f92c5670(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:21,108 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:21,108 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=12, startTime=1732619841015; duration=0sec 2024-11-26T11:17:21,108 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:21,108 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:21,108 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T11:17:21,110 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T11:17:21,110 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:21,110 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:21,110 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1348b9ed87484f7293038c6acb98ff26, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/21b0bf501a5f422d8ca24472f76b5031, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/8f2954257fde433999930566bae7ffbf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/3f580d91184e4c96960ce4130d49a9c2] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=48.5 K 2024-11-26T11:17:21,111 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1348b9ed87484f7293038c6acb98ff26, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732619836792 2024-11-26T11:17:21,114 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b0bf501a5f422d8ca24472f76b5031, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732619838151 2024-11-26T11:17:21,115 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f2954257fde433999930566bae7ffbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732619838867 2024-11-26T11:17:21,116 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f580d91184e4c96960ce4130d49a9c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732619840089 2024-11-26T11:17:21,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742401_1577 (size=4469) 2024-11-26T11:17:21,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112666d86c305dad4629a96c418de3d7a274_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619840191/Put/seqid=0 2024-11-26T11:17:21,163 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:21,163 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/1f046dbdef9947a7adf8ea179be75567 is 50, key is test_row_0/C:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:21,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742403_1579 (size=12915) 2024-11-26T11:17:21,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742402_1578 (size=12454) 2024-11-26T11:17:21,190 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/1f046dbdef9947a7adf8ea179be75567 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1f046dbdef9947a7adf8ea179be75567 2024-11-26T11:17:21,198 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into 1f046dbdef9947a7adf8ea179be75567(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:21,198 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:21,198 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=12, startTime=1732619841015; duration=0sec 2024-11-26T11:17:21,198 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:21,198 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T11:17:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:21,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619901400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619901513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:21,541 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#481 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:21,542 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/479f4a3ad7504962abfade1db579ed65 is 175, key is test_row_0/A:col10/1732619840103/Put/seqid=0 2024-11-26T11:17:21,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742404_1580 (size=31869) 2024-11-26T11:17:21,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:21,588 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112666d86c305dad4629a96c418de3d7a274_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666d86c305dad4629a96c418de3d7a274_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/635d9878e8b64a50bb22ec1b60d1b1cf, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:21,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/635d9878e8b64a50bb22ec1b60d1b1cf is 175, key is test_row_0/A:col10/1732619840191/Put/seqid=0 2024-11-26T11:17:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742405_1581 (size=31255) 2024-11-26T11:17:21,598 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/635d9878e8b64a50bb22ec1b60d1b1cf 2024-11-26T11:17:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/eca3ffb837d743b491d84425daf7dd90 is 50, key is test_row_0/B:col10/1732619840191/Put/seqid=0 2024-11-26T11:17:21,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742406_1582 (size=12301) 2024-11-26T11:17:21,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619901723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:21,952 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/479f4a3ad7504962abfade1db579ed65 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/479f4a3ad7504962abfade1db579ed65 2024-11-26T11:17:21,956 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into 479f4a3ad7504962abfade1db579ed65(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:21,956 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:21,956 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=12, startTime=1732619841015; duration=0sec 2024-11-26T11:17:21,957 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:21,957 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:22,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48302 deadline: 1732619902003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:22,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48254 deadline: 1732619902003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:22,004 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:22,005 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_1' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:22,016 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/eca3ffb837d743b491d84425daf7dd90 2024-11-26T11:17:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/c4fc8c8928cf48d79df235c417bc5b03 is 50, key is test_row_0/C:col10/1732619840191/Put/seqid=0 2024-11-26T11:17:22,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619902032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:22,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742407_1583 (size=12301) 2024-11-26T11:17:22,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/c4fc8c8928cf48d79df235c417bc5b03 2024-11-26T11:17:22,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/635d9878e8b64a50bb22ec1b60d1b1cf as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf 2024-11-26T11:17:22,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf, entries=150, sequenceid=393, filesize=30.5 K 2024-11-26T11:17:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/eca3ffb837d743b491d84425daf7dd90 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/eca3ffb837d743b491d84425daf7dd90 2024-11-26T11:17:22,051 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/eca3ffb837d743b491d84425daf7dd90, entries=150, sequenceid=393, filesize=12.0 K 2024-11-26T11:17:22,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/c4fc8c8928cf48d79df235c417bc5b03 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/c4fc8c8928cf48d79df235c417bc5b03 2024-11-26T11:17:22,055 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/c4fc8c8928cf48d79df235c417bc5b03, entries=150, sequenceid=393, filesize=12.0 K 2024-11-26T11:17:22,056 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fa0c0e175d4b592132f0942e2a244b29 in 962ms, sequenceid=393, compaction requested=false 2024-11-26T11:17:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-26T11:17:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-26T11:17:22,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-26T11:17:22,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-26T11:17:22,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8880 sec 2024-11-26T11:17:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T11:17:22,324 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:22,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-26T11:17:22,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:22,340 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:22,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T11:17:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T11:17:22,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T11:17:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:22,494 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:17:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:22,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266093dec584c34cdd87730b49b2082a0b_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619841398/Put/seqid=0 2024-11-26T11:17:22,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742408_1584 (size=12454) 2024-11-26T11:17:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:22,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619902596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T11:17:22,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619902704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:22,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619902913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:22,928 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266093dec584c34cdd87730b49b2082a0b_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266093dec584c34cdd87730b49b2082a0b_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:22,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/767beef559644f8cb8d24356bf872e61, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:22,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/767beef559644f8cb8d24356bf872e61 is 175, key is test_row_0/A:col10/1732619841398/Put/seqid=0 2024-11-26T11:17:22,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742409_1585 (size=31255) 2024-11-26T11:17:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T11:17:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619903223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:23,333 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/767beef559644f8cb8d24356bf872e61 2024-11-26T11:17:23,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/8d03c9d58a8f4775a3bf036b9cc3f494 is 50, key is test_row_0/B:col10/1732619841398/Put/seqid=0 2024-11-26T11:17:23,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742410_1586 (size=12301) 2024-11-26T11:17:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T11:17:23,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619903733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:23,745 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/8d03c9d58a8f4775a3bf036b9cc3f494 2024-11-26T11:17:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/327604a2f7f84bfaacbbae9a3dd8da02 is 50, key is test_row_0/C:col10/1732619841398/Put/seqid=0 2024-11-26T11:17:23,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742411_1587 (size=12301) 2024-11-26T11:17:23,783 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/327604a2f7f84bfaacbbae9a3dd8da02 2024-11-26T11:17:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/767beef559644f8cb8d24356bf872e61 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61 2024-11-26T11:17:23,793 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61, entries=150, sequenceid=413, filesize=30.5 K 2024-11-26T11:17:23,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/8d03c9d58a8f4775a3bf036b9cc3f494 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/8d03c9d58a8f4775a3bf036b9cc3f494 2024-11-26T11:17:23,796 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/8d03c9d58a8f4775a3bf036b9cc3f494, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T11:17:23,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/327604a2f7f84bfaacbbae9a3dd8da02 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/327604a2f7f84bfaacbbae9a3dd8da02 2024-11-26T11:17:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,802 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/327604a2f7f84bfaacbbae9a3dd8da02, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T11:17:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,803 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fa0c0e175d4b592132f0942e2a244b29 in 1309ms, sequenceid=413, compaction requested=true 2024-11-26T11:17:23,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:23,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:23,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-26T11:17:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-26T11:17:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-26T11:17:23,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-11-26T11:17:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4830 sec 2024-11-26T11:17:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T11:17:24,503 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:24,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-26T11:17:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T11:17:24,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:24,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:24,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T11:17:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T11:17:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:24,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T11:17:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260e41a978571e4eb5b2bd1dbf793cba1c_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619842578/Put/seqid=0 2024-11-26T11:17:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742412_1588 (size=9914) 2024-11-26T11:17:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T11:17:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619904838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619904952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:25,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48258 deadline: 1732619905003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:25,007 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_0' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:25,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48294 deadline: 1732619905012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:25,016 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7f3a3edaecc3,37499,1732619676667 for 'test_row_2' in fa0c0e175d4b592132f0942e2a244b29 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T11:17:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:25,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:25,074 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260e41a978571e4eb5b2bd1dbf793cba1c_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e41a978571e4eb5b2bd1dbf793cba1c_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:25,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/e93cb63ce91b4078b5007f8e0dc13a16, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:25,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/e93cb63ce91b4078b5007f8e0dc13a16 is 175, key is test_row_0/A:col10/1732619842578/Put/seqid=0 2024-11-26T11:17:25,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742413_1589 (size=22561) 2024-11-26T11:17:25,137 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/e93cb63ce91b4078b5007f8e0dc13a16 2024-11-26T11:17:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T11:17:25,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/e8973dd6f62b4f8ab946538b888f4a67 is 50, key is test_row_0/B:col10/1732619842578/Put/seqid=0 2024-11-26T11:17:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619905162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742414_1590 (size=9857) 2024-11-26T11:17:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:25,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/e8973dd6f62b4f8ab946538b888f4a67 2024-11-26T11:17:25,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/9a03758cb3ac4bf495f72250ab7f4443 is 50, key is test_row_0/C:col10/1732619842578/Put/seqid=0 2024-11-26T11:17:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742415_1591 (size=9857) 2024-11-26T11:17:25,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619905472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:25,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/9a03758cb3ac4bf495f72250ab7f4443 2024-11-26T11:17:25,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/e93cb63ce91b4078b5007f8e0dc13a16 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16 2024-11-26T11:17:25,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16, entries=100, sequenceid=428, filesize=22.0 K 2024-11-26T11:17:25,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/e8973dd6f62b4f8ab946538b888f4a67 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e8973dd6f62b4f8ab946538b888f4a67 2024-11-26T11:17:25,607 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e8973dd6f62b4f8ab946538b888f4a67, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T11:17:25,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/9a03758cb3ac4bf495f72250ab7f4443 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/9a03758cb3ac4bf495f72250ab7f4443 2024-11-26T11:17:25,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/9a03758cb3ac4bf495f72250ab7f4443, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T11:17:25,617 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fa0c0e175d4b592132f0942e2a244b29 in 959ms, sequenceid=428, compaction requested=true 2024-11-26T11:17:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-26T11:17:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-26T11:17:25,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-26T11:17:25,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1140 sec 2024-11-26T11:17:25,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.1190 sec 2024-11-26T11:17:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T11:17:25,653 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:25,653 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-26T11:17:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T11:17:25,655 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:25,656 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:25,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T11:17:25,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:25,809 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266c8eece0f96b4fc3bbb847f07ee44524_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619844831/Put/seqid=0 2024-11-26T11:17:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742416_1592 (size=12454) 2024-11-26T11:17:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T11:17:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:25,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619906039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:26,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619906153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:26,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:26,224 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266c8eece0f96b4fc3bbb847f07ee44524_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266c8eece0f96b4fc3bbb847f07ee44524_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:26,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/86b1f423362b4997a4fd4ade8f78fe8f, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/86b1f423362b4997a4fd4ade8f78fe8f is 175, key is test_row_0/A:col10/1732619844831/Put/seqid=0 2024-11-26T11:17:26,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742417_1593 (size=31255) 2024-11-26T11:17:26,241 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/86b1f423362b4997a4fd4ade8f78fe8f 2024-11-26T11:17:26,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/e898ae35b1cb4ed2981e436f56cc860b is 50, key is test_row_0/B:col10/1732619844831/Put/seqid=0 2024-11-26T11:17:26,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742418_1594 (size=12301) 2024-11-26T11:17:26,258 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/e898ae35b1cb4ed2981e436f56cc860b 2024-11-26T11:17:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/ee783a6fc0074cc6bd97d7aca6f91de3 is 50, key is test_row_0/C:col10/1732619844831/Put/seqid=0 2024-11-26T11:17:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742419_1595 (size=12301) 2024-11-26T11:17:26,274 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/ee783a6fc0074cc6bd97d7aca6f91de3 2024-11-26T11:17:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/86b1f423362b4997a4fd4ade8f78fe8f as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f 2024-11-26T11:17:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T11:17:26,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f, entries=150, sequenceid=445, filesize=30.5 K 2024-11-26T11:17:26,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/e898ae35b1cb4ed2981e436f56cc860b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e898ae35b1cb4ed2981e436f56cc860b 2024-11-26T11:17:26,289 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e898ae35b1cb4ed2981e436f56cc860b, entries=150, sequenceid=445, filesize=12.0 K 2024-11-26T11:17:26,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/ee783a6fc0074cc6bd97d7aca6f91de3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/ee783a6fc0074cc6bd97d7aca6f91de3 2024-11-26T11:17:26,293 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/ee783a6fc0074cc6bd97d7aca6f91de3, entries=150, sequenceid=445, filesize=12.0 K 2024-11-26T11:17:26,293 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fa0c0e175d4b592132f0942e2a244b29 in 484ms, sequenceid=445, compaction requested=true 2024-11-26T11:17:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-26T11:17:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-26T11:17:26,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-26T11:17:26,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 639 msec 2024-11-26T11:17:26,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 644 msec 2024-11-26T11:17:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:26,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:26,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265d6acd0bef54491eb823b1cf563ff51a_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:26,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742420_1596 (size=12454) 2024-11-26T11:17:26,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619906476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:26,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619906593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T11:17:26,793 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:26,793 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T11:17:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-26T11:17:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:26,796 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T11:17:26,797 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T11:17:26,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T11:17:26,804 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:26,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619906807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:26,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265d6acd0bef54491eb823b1cf563ff51a_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265d6acd0bef54491eb823b1cf563ff51a_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:26,809 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d5910e634c6446768bb8e927adee6ea6, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:26,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d5910e634c6446768bb8e927adee6ea6 is 175, key is test_row_0/A:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:26,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742421_1597 (size=31255) 2024-11-26T11:17:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:26,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:26,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619907121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:27,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d5910e634c6446768bb8e927adee6ea6 2024-11-26T11:17:27,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/aafe370facdf40edafe4d9aede4ac7a5 is 50, key is test_row_0/B:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:27,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742422_1598 (size=12301) 2024-11-26T11:17:27,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:27,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:27,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:27,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619907642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:27,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/aafe370facdf40edafe4d9aede4ac7a5 2024-11-26T11:17:27,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/f5c4a53547924fefa119d35266e82135 is 50, key is test_row_0/C:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:27,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742423_1599 (size=12301) 2024-11-26T11:17:27,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:27,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:28,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:28,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:28,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:28,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T11:17:28,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/f5c4a53547924fefa119d35266e82135 2024-11-26T11:17:28,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d5910e634c6446768bb8e927adee6ea6 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6 2024-11-26T11:17:28,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6, entries=150, sequenceid=461, filesize=30.5 K 2024-11-26T11:17:28,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/aafe370facdf40edafe4d9aede4ac7a5 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/aafe370facdf40edafe4d9aede4ac7a5 2024-11-26T11:17:28,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/aafe370facdf40edafe4d9aede4ac7a5, entries=150, sequenceid=461, filesize=12.0 K 2024-11-26T11:17:28,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/f5c4a53547924fefa119d35266e82135 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/f5c4a53547924fefa119d35266e82135 2024-11-26T11:17:28,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/f5c4a53547924fefa119d35266e82135, entries=150, sequenceid=461, filesize=12.0 K 2024-11-26T11:17:28,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fa0c0e175d4b592132f0942e2a244b29 in 1747ms, sequenceid=461, compaction requested=true 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:28,114 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:28,114 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T11:17:28,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:28,115 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 179450 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T11:17:28,115 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71976 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T11:17:28,115 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:28,115 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:28,115 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,115 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,116 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/479f4a3ad7504962abfade1db579ed65, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=175.2 K 2024-11-26T11:17:28,116 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/6d09695c4b7e486aab9d3610f92c5670, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/eca3ffb837d743b491d84425daf7dd90, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/8d03c9d58a8f4775a3bf036b9cc3f494, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e8973dd6f62b4f8ab946538b888f4a67, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e898ae35b1cb4ed2981e436f56cc860b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/aafe370facdf40edafe4d9aede4ac7a5] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=70.3 K 2024-11-26T11:17:28,116 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/479f4a3ad7504962abfade1db579ed65, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6] 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479f4a3ad7504962abfade1db579ed65, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732619840089 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d09695c4b7e486aab9d3610f92c5670, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732619840089 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 635d9878e8b64a50bb22ec1b60d1b1cf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732619840170 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting eca3ffb837d743b491d84425daf7dd90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732619840170 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 767beef559644f8cb8d24356bf872e61, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732619841370 2024-11-26T11:17:28,116 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d03c9d58a8f4775a3bf036b9cc3f494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732619841370 2024-11-26T11:17:28,117 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e8973dd6f62b4f8ab946538b888f4a67, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732619842578 2024-11-26T11:17:28,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting e93cb63ce91b4078b5007f8e0dc13a16, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732619842578 2024-11-26T11:17:28,117 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting e898ae35b1cb4ed2981e436f56cc860b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732619844826 2024-11-26T11:17:28,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86b1f423362b4997a4fd4ade8f78fe8f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732619844826 2024-11-26T11:17:28,117 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting aafe370facdf40edafe4d9aede4ac7a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732619846018 2024-11-26T11:17:28,117 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5910e634c6446768bb8e927adee6ea6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732619846018 2024-11-26T11:17:28,134 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:28,135 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/67d132aee3c0425a8ecd3fa45691fdd2 is 50, key is test_row_0/B:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:28,136 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742424_1600 (size=13119) 2024-11-26T11:17:28,145 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126bbdcbe215bc5422dbc14daa6e539079d_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:28,148 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126bbdcbe215bc5422dbc14daa6e539079d_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:28,148 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bbdcbe215bc5422dbc14daa6e539079d_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:28,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742425_1601 (size=4469) 2024-11-26T11:17:28,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37499 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T11:17:28,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,206 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T11:17:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:28,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:28,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a750d523ae147db9d3d77f9053c4e22_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619846398/Put/seqid=0 2024-11-26T11:17:28,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742426_1602 (size=12454) 2024-11-26T11:17:28,545 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/67d132aee3c0425a8ecd3fa45691fdd2 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/67d132aee3c0425a8ecd3fa45691fdd2 2024-11-26T11:17:28,548 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 67d132aee3c0425a8ecd3fa45691fdd2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:28,548 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:28,548 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=10, startTime=1732619848114; duration=0sec 2024-11-26T11:17:28,548 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:28,548 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:28,548 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T11:17:28,550 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71976 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T11:17:28,550 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:28,550 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:28,550 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1f046dbdef9947a7adf8ea179be75567, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/c4fc8c8928cf48d79df235c417bc5b03, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/327604a2f7f84bfaacbbae9a3dd8da02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/9a03758cb3ac4bf495f72250ab7f4443, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/ee783a6fc0074cc6bd97d7aca6f91de3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/f5c4a53547924fefa119d35266e82135] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=70.3 K 2024-11-26T11:17:28,552 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f046dbdef9947a7adf8ea179be75567, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732619840089 2024-11-26T11:17:28,552 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting c4fc8c8928cf48d79df235c417bc5b03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732619840170 2024-11-26T11:17:28,553 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 327604a2f7f84bfaacbbae9a3dd8da02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732619841370 2024-11-26T11:17:28,553 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:28,554 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/ea9b581d40cb4ca7b4f3f048cc51173c is 175, key is test_row_0/A:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:28,554 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a03758cb3ac4bf495f72250ab7f4443, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732619842578 2024-11-26T11:17:28,554 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ee783a6fc0074cc6bd97d7aca6f91de3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732619844826 2024-11-26T11:17:28,555 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c4a53547924fefa119d35266e82135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732619846018 2024-11-26T11:17:28,576 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:28,576 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/776538b770cd4bc49b9b00b1e8a5f86a is 50, key is test_row_0/C:col10/1732619846365/Put/seqid=0 2024-11-26T11:17:28,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742427_1603 (size=32073) 2024-11-26T11:17:28,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742428_1604 (size=13119) 2024-11-26T11:17:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:28,621 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a750d523ae147db9d3d77f9053c4e22_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a750d523ae147db9d3d77f9053c4e22_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8cac9d9802024cca92cf2f94ac795a15, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:28,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8cac9d9802024cca92cf2f94ac795a15 is 175, key is test_row_0/A:col10/1732619846398/Put/seqid=0 2024-11-26T11:17:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742429_1605 (size=31255) 2024-11-26T11:17:28,639 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8cac9d9802024cca92cf2f94ac795a15 2024-11-26T11:17:28,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/ff25b07c7b494f2787309fc77428b677 is 50, key is test_row_0/B:col10/1732619846398/Put/seqid=0 2024-11-26T11:17:28,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742430_1606 (size=12301) 2024-11-26T11:17:28,652 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/ff25b07c7b494f2787309fc77428b677 2024-11-26T11:17:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/b86a8cbee0184f839c1a49365cbd7527 is 50, key is test_row_0/C:col10/1732619846398/Put/seqid=0 2024-11-26T11:17:28,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. as already flushing 2024-11-26T11:17:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:28,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742431_1607 (size=12301) 2024-11-26T11:17:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619908731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:28,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619908843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:29,000 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/ea9b581d40cb4ca7b4f3f048cc51173c as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/ea9b581d40cb4ca7b4f3f048cc51173c 2024-11-26T11:17:29,007 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/776538b770cd4bc49b9b00b1e8a5f86a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/776538b770cd4bc49b9b00b1e8a5f86a 2024-11-26T11:17:29,008 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into ea9b581d40cb4ca7b4f3f048cc51173c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:29,008 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:29,008 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=10, startTime=1732619848114; duration=0sec 2024-11-26T11:17:29,008 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:29,008 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:29,015 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into 776538b770cd4bc49b9b00b1e8a5f86a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:29,015 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:29,015 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=10, startTime=1732619848114; duration=0sec 2024-11-26T11:17:29,015 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:29,015 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T11:17:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37499 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48274 deadline: 1732619909053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 , the old value is region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T11:17:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., hostname=7f3a3edaecc3,37499,1732619676667, seqNum=5 because the exception is null or not the one we care about 2024-11-26T11:17:29,085 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/b86a8cbee0184f839c1a49365cbd7527 2024-11-26T11:17:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/8cac9d9802024cca92cf2f94ac795a15 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15 2024-11-26T11:17:29,092 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15, entries=150, sequenceid=477, filesize=30.5 K 2024-11-26T11:17:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/ff25b07c7b494f2787309fc77428b677 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ff25b07c7b494f2787309fc77428b677 2024-11-26T11:17:29,095 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ff25b07c7b494f2787309fc77428b677, entries=150, sequenceid=477, filesize=12.0 K 2024-11-26T11:17:29,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/b86a8cbee0184f839c1a49365cbd7527 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b86a8cbee0184f839c1a49365cbd7527 2024-11-26T11:17:29,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b86a8cbee0184f839c1a49365cbd7527, entries=150, sequenceid=477, filesize=12.0 K 2024-11-26T11:17:29,099 INFO [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fa0c0e175d4b592132f0942e2a244b29 in 893ms, sequenceid=477, compaction requested=false 2024-11-26T11:17:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7f3a3edaecc3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-26T11:17:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-26T11:17:29,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-26T11:17:29,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3030 sec 2024-11-26T11:17:29,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.3090 sec 2024-11-26T11:17:29,292 INFO [Thread-2274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2274. 2024-11-26T11:17:29,293 DEBUG [Thread-2274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:29,293 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:29,293 INFO [Thread-2268 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2268. 2024-11-26T11:17:29,293 DEBUG [Thread-2268 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:29,293 DEBUG [Thread-2268 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,293 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,293 DEBUG [Thread-2268 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:29,295 INFO [Thread-2272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2272. 2024-11-26T11:17:29,295 DEBUG [Thread-2272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:29,295 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:29,296 INFO [Thread-2266 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2266. 2024-11-26T11:17:29,296 DEBUG [Thread-2266 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:29,296 DEBUG [Thread-2266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,296 DEBUG [Thread-2266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,296 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:29,298 INFO [Thread-2270 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2270. 2024-11-26T11:17:29,298 DEBUG [Thread-2270 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:29,298 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,298 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37499 {}] regionserver.HRegion(8855): Flush requested on fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:29,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T11:17:29,364 INFO [Thread-2261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2261. 2024-11-26T11:17:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:29,364 DEBUG [Thread-2261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:29,364 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:29,364 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:29,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:29,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126026e9a56934a4edd8b94e77541f586ce_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:29,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742432_1608 (size=12454) 2024-11-26T11:17:29,774 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:29,778 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126026e9a56934a4edd8b94e77541f586ce_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126026e9a56934a4edd8b94e77541f586ce_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:29,778 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/50fb228a446344b5863708a871508d42, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:29,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/50fb228a446344b5863708a871508d42 is 175, key is test_row_0/A:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:29,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742433_1609 (size=31255) 2024-11-26T11:17:30,193 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=497, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/50fb228a446344b5863708a871508d42 2024-11-26T11:17:30,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/311d498d93294ba0883d64d480c1f3e8 is 50, key is test_row_0/B:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:30,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742434_1610 (size=12301) 2024-11-26T11:17:30,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/311d498d93294ba0883d64d480c1f3e8 2024-11-26T11:17:30,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/96104f97ed814a439030f925b4522052 is 50, key is test_row_0/C:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:30,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742435_1611 (size=12301) 2024-11-26T11:17:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T11:17:30,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:31,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/96104f97ed814a439030f925b4522052 2024-11-26T11:17:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/50fb228a446344b5863708a871508d42 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42 2024-11-26T11:17:31,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42, entries=150, sequenceid=497, filesize=30.5 K 2024-11-26T11:17:31,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/311d498d93294ba0883d64d480c1f3e8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/311d498d93294ba0883d64d480c1f3e8 2024-11-26T11:17:31,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/311d498d93294ba0883d64d480c1f3e8, entries=150, sequenceid=497, filesize=12.0 K 2024-11-26T11:17:31,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/96104f97ed814a439030f925b4522052 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/96104f97ed814a439030f925b4522052 2024-11-26T11:17:31,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/96104f97ed814a439030f925b4522052, entries=150, sequenceid=497, filesize=12.0 K 2024-11-26T11:17:31,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for fa0c0e175d4b592132f0942e2a244b29 in 1669ms, sequenceid=497, compaction requested=true 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:31,034 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa0c0e175d4b592132f0942e2a244b29:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T11:17:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:31,034 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:31,034 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/A is initiating minor compaction (all files) 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:31,035 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/A in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/B is initiating minor compaction (all files) 2024-11-26T11:17:31,035 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/ea9b581d40cb4ca7b4f3f048cc51173c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=92.4 K 2024-11-26T11:17:31,035 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:31,035 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/B in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. files: [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/ea9b581d40cb4ca7b4f3f048cc51173c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42] 2024-11-26T11:17:31,035 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/67d132aee3c0425a8ecd3fa45691fdd2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ff25b07c7b494f2787309fc77428b677, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/311d498d93294ba0883d64d480c1f3e8] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=36.8 K 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea9b581d40cb4ca7b4f3f048cc51173c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732619846018 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cac9d9802024cca92cf2f94ac795a15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732619846398 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d132aee3c0425a8ecd3fa45691fdd2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732619846018 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50fb228a446344b5863708a871508d42, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732619848692 2024-11-26T11:17:31,035 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting ff25b07c7b494f2787309fc77428b677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732619846398 2024-11-26T11:17:31,036 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 311d498d93294ba0883d64d480c1f3e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732619848692 2024-11-26T11:17:31,039 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:31,040 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126d9dcfb46993b42e4b8acba361b93b396_fa0c0e175d4b592132f0942e2a244b29 store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:31,041 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:31,041 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/862f65fd93a7493b86208163034aabbc is 50, key is test_row_0/B:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:31,043 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126d9dcfb46993b42e4b8acba361b93b396_fa0c0e175d4b592132f0942e2a244b29, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:31,043 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d9dcfb46993b42e4b8acba361b93b396_fa0c0e175d4b592132f0942e2a244b29 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:31,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742436_1612 (size=13221) 2024-11-26T11:17:31,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742437_1613 (size=4469) 2024-11-26T11:17:31,047 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/862f65fd93a7493b86208163034aabbc as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/862f65fd93a7493b86208163034aabbc 2024-11-26T11:17:31,051 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/B of fa0c0e175d4b592132f0942e2a244b29 into 862f65fd93a7493b86208163034aabbc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:31,051 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:31,051 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/B, priority=13, startTime=1732619851034; duration=0sec 2024-11-26T11:17:31,051 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T11:17:31,051 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:B 2024-11-26T11:17:31,051 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T11:17:31,052 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T11:17:31,052 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1541): fa0c0e175d4b592132f0942e2a244b29/C is initiating minor compaction (all files) 2024-11-26T11:17:31,052 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa0c0e175d4b592132f0942e2a244b29/C in TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:31,052 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/776538b770cd4bc49b9b00b1e8a5f86a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b86a8cbee0184f839c1a49365cbd7527, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/96104f97ed814a439030f925b4522052] into tmpdir=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp, totalSize=36.8 K 2024-11-26T11:17:31,052 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 776538b770cd4bc49b9b00b1e8a5f86a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732619846018 2024-11-26T11:17:31,052 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting b86a8cbee0184f839c1a49365cbd7527, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732619846398 2024-11-26T11:17:31,053 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] compactions.Compactor(225): Compacting 96104f97ed814a439030f925b4522052, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732619848692 2024-11-26T11:17:31,058 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:31,059 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/adb3d53868dd4c78aba87436767572e3 is 50, key is test_row_0/C:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:31,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742438_1614 (size=13221) 2024-11-26T11:17:31,067 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/adb3d53868dd4c78aba87436767572e3 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/adb3d53868dd4c78aba87436767572e3 2024-11-26T11:17:31,070 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/C of fa0c0e175d4b592132f0942e2a244b29 into adb3d53868dd4c78aba87436767572e3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:31,070 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:31,070 INFO [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/C, priority=13, startTime=1732619851034; duration=0sec 2024-11-26T11:17:31,070 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:31,070 DEBUG [RS:0;7f3a3edaecc3:37499-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:C 2024-11-26T11:17:31,447 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa0c0e175d4b592132f0942e2a244b29#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T11:17:31,447 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d651be8833384ff9bc8bdcde9c44ab6a is 175, key is test_row_0/A:col10/1732619848729/Put/seqid=0 2024-11-26T11:17:31,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742439_1615 (size=32175) 2024-11-26T11:17:31,855 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/d651be8833384ff9bc8bdcde9c44ab6a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d651be8833384ff9bc8bdcde9c44ab6a 2024-11-26T11:17:31,859 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa0c0e175d4b592132f0942e2a244b29/A of fa0c0e175d4b592132f0942e2a244b29 into d651be8833384ff9bc8bdcde9c44ab6a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T11:17:31,859 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa0c0e175d4b592132f0942e2a244b29: 2024-11-26T11:17:31,859 INFO [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., storeName=fa0c0e175d4b592132f0942e2a244b29/A, priority=13, startTime=1732619851034; duration=0sec 2024-11-26T11:17:31,859 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T11:17:31,859 DEBUG [RS:0;7f3a3edaecc3:37499-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa0c0e175d4b592132f0942e2a244b29:A 2024-11-26T11:17:32,064 INFO [Thread-2263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2263. 2024-11-26T11:17:32,064 DEBUG [Thread-2263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:32,064 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:32,064 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:32,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:32,105 INFO [Thread-2259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2259. 2024-11-26T11:17:32,105 DEBUG [Thread-2259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:32,105 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:32,105 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:32,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:34,264 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T11:17:35,034 INFO [Thread-2255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2255. 2024-11-26T11:17:35,034 DEBUG [Thread-2255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:35,034 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:35,034 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:35,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:35,116 INFO [Thread-2257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2257. 2024-11-26T11:17:35,116 DEBUG [Thread-2257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T11:17:35,116 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:35,117 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:35,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 188 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2295 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2249 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2293 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2272 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2245 2024-11-26T11:17:35,117 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T11:17:35,118 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:17:35,118 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:17:35,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:35,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:35,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:35,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T11:17:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:35,123 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619855123"}]},"ts":"1732619855123"} 2024-11-26T11:17:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T11:17:35,127 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T11:17:35,127 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T11:17:35,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T11:17:35,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, UNASSIGN}] 2024-11-26T11:17:35,131 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, UNASSIGN 2024-11-26T11:17:35,132 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=CLOSING, regionLocation=7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:35,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, UNASSIGN because future has completed 2024-11-26T11:17:35,133 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T11:17:35,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667}] 2024-11-26T11:17:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T11:17:35,286 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:35,286 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T11:17:35,286 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing fa0c0e175d4b592132f0942e2a244b29, disabling compactions & flushes 2024-11-26T11:17:35,286 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:35,286 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:35,286 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. after waiting 0 ms 2024-11-26T11:17:35,286 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:35,286 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing fa0c0e175d4b592132f0942e2a244b29 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T11:17:35,287 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=A 2024-11-26T11:17:35,287 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:35,287 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=B 2024-11-26T11:17:35,287 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:35,287 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa0c0e175d4b592132f0942e2a244b29, store=C 2024-11-26T11:17:35,287 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T11:17:35,294 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126afd5052d157440b2a37b820912cd6ae0_fa0c0e175d4b592132f0942e2a244b29 is 50, key is test_row_0/A:col10/1732619855033/Put/seqid=0 2024-11-26T11:17:35,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742440_1616 (size=12454) 2024-11-26T11:17:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T11:17:35,699 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T11:17:35,709 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126afd5052d157440b2a37b820912cd6ae0_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126afd5052d157440b2a37b820912cd6ae0_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:35,711 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/26d58a23f4ec4aceb3198584136a2aa4, store: [table=TestAcidGuarantees family=A region=fa0c0e175d4b592132f0942e2a244b29] 2024-11-26T11:17:35,712 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/26d58a23f4ec4aceb3198584136a2aa4 is 175, key is test_row_0/A:col10/1732619855033/Put/seqid=0 2024-11-26T11:17:35,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742441_1617 (size=31255) 2024-11-26T11:17:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T11:17:36,117 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=507, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/26d58a23f4ec4aceb3198584136a2aa4 2024-11-26T11:17:36,129 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/023c9e6e9d21453789e788aac7237a44 is 50, key is test_row_0/B:col10/1732619855033/Put/seqid=0 2024-11-26T11:17:36,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742442_1618 (size=12301) 2024-11-26T11:17:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T11:17:36,535 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/023c9e6e9d21453789e788aac7237a44 2024-11-26T11:17:36,549 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/e0e932609b32414b89a9162896d8cb70 is 50, key is test_row_0/C:col10/1732619855033/Put/seqid=0 2024-11-26T11:17:36,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742443_1619 (size=12301) 2024-11-26T11:17:36,955 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/e0e932609b32414b89a9162896d8cb70 2024-11-26T11:17:36,963 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/A/26d58a23f4ec4aceb3198584136a2aa4 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/26d58a23f4ec4aceb3198584136a2aa4 2024-11-26T11:17:36,966 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/26d58a23f4ec4aceb3198584136a2aa4, entries=150, sequenceid=507, filesize=30.5 K 2024-11-26T11:17:36,968 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/B/023c9e6e9d21453789e788aac7237a44 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/023c9e6e9d21453789e788aac7237a44 2024-11-26T11:17:36,971 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/023c9e6e9d21453789e788aac7237a44, entries=150, sequenceid=507, filesize=12.0 K 2024-11-26T11:17:36,972 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/.tmp/C/e0e932609b32414b89a9162896d8cb70 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/e0e932609b32414b89a9162896d8cb70 2024-11-26T11:17:36,975 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/e0e932609b32414b89a9162896d8cb70, entries=150, sequenceid=507, filesize=12.0 K 2024-11-26T11:17:36,975 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fa0c0e175d4b592132f0942e2a244b29 in 1689ms, sequenceid=507, compaction requested=false 2024-11-26T11:17:36,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/45beab274ec24392a6507efe73bb9394, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/48b4fae6dac8497fa9655da7d41f53f2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f92fd9ccb0e0434db8d7f46b03cf1e06, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bbb2cd4a0fe94efe85365bfe478d68c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/479f4a3ad7504962abfade1db579ed65, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/ea9b581d40cb4ca7b4f3f048cc51173c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42] to archive 2024-11-26T11:17:36,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:17:36,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/4a6a9e9c52b64387a9fc2300ebc94194 2024-11-26T11:17:36,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d8f15cbe89a54b7bab5aa69d8943c17c 2024-11-26T11:17:36,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/22ebd063f0f34e66a93881f7fa3b54c6 2024-11-26T11:17:36,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/45beab274ec24392a6507efe73bb9394 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/45beab274ec24392a6507efe73bb9394 2024-11-26T11:17:36,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/43fdf7a2909a4c3db12b69001d68b8ce 2024-11-26T11:17:36,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/343566814b284001ab03e8c357be5ae3 2024-11-26T11:17:36,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/c51cd9695f3643ceb0b99e8d82815ad9 2024-11-26T11:17:36,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/51588c7f375d400ea13a5d8db74ae87e 2024-11-26T11:17:36,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/48b4fae6dac8497fa9655da7d41f53f2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/48b4fae6dac8497fa9655da7d41f53f2 2024-11-26T11:17:36,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/11476c431b484addb3bb9eb8709a0160 2024-11-26T11:17:36,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bd8c89894b484f7bbe6c9af5ae9021f0 2024-11-26T11:17:36,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f92fd9ccb0e0434db8d7f46b03cf1e06 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f92fd9ccb0e0434db8d7f46b03cf1e06 2024-11-26T11:17:36,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8b202f9aed134631b34db39066735a4e 2024-11-26T11:17:36,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bbb2cd4a0fe94efe85365bfe478d68c3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/bbb2cd4a0fe94efe85365bfe478d68c3 2024-11-26T11:17:36,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/0df1f3c87be24c6390fc5b58623a76be 2024-11-26T11:17:36,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/faa263ff1bb64a9fb71435c6fe9e49a1 2024-11-26T11:17:36,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/f5c413e7eb0c4ce787cdb5e5f04b65c3 2024-11-26T11:17:36,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/a3e2fed278b84bfe877266ed2a420610 2024-11-26T11:17:36,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/479f4a3ad7504962abfade1db579ed65 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/479f4a3ad7504962abfade1db579ed65 2024-11-26T11:17:36,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/635d9878e8b64a50bb22ec1b60d1b1cf 2024-11-26T11:17:36,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/767beef559644f8cb8d24356bf872e61 2024-11-26T11:17:37,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/e93cb63ce91b4078b5007f8e0dc13a16 2024-11-26T11:17:37,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/86b1f423362b4997a4fd4ade8f78fe8f 2024-11-26T11:17:37,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/ea9b581d40cb4ca7b4f3f048cc51173c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/ea9b581d40cb4ca7b4f3f048cc51173c 2024-11-26T11:17:37,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d5910e634c6446768bb8e927adee6ea6 2024-11-26T11:17:37,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/8cac9d9802024cca92cf2f94ac795a15 2024-11-26T11:17:37,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/50fb228a446344b5863708a871508d42 2024-11-26T11:17:37,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/53b1954aec65410e8abae7b43cf38570, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/409d2818fd174d6ea2f8bb53ad3e898b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/239d72ad3bbd42feb367acf36c4a4826, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0827f33f6bf443ebb4eeb5c455736d62, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1e1716b290fe431e8e134e66dc13b47f, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/c758c03342084b1ba69b6e1f9254e259, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/4583de94b37843e2b305e700ae083cef, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/137cd646406b42c28d1681b0f7fe436c, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0cdcd7b2e0624932b286a7f4c14be3cc, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1789c6199d4941679e2aa2430bb8b58e, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/079842749cff42bea7b344bdff8d4151, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/08d18551510b4d7490aa94b902d483c3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/af6e905867ef42d88a94c8a9986c7e9d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/83813c9b862242448bbdfbaad2d60a43, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/945d641ce9754e7e9502bbf7658211e1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ab0484a4a49b435881da4e23f907f061, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/bbd3c3e783ea41139fa39225f00b55eb, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/6d09695c4b7e486aab9d3610f92c5670, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/b7cee3e571394c2482557adb67b7b6b6, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/eca3ffb837d743b491d84425daf7dd90, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/8d03c9d58a8f4775a3bf036b9cc3f494, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e8973dd6f62b4f8ab946538b888f4a67, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e898ae35b1cb4ed2981e436f56cc860b, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/67d132aee3c0425a8ecd3fa45691fdd2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/aafe370facdf40edafe4d9aede4ac7a5, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ff25b07c7b494f2787309fc77428b677, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/311d498d93294ba0883d64d480c1f3e8] to archive 2024-11-26T11:17:37,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:17:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/53b1954aec65410e8abae7b43cf38570 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/53b1954aec65410e8abae7b43cf38570 2024-11-26T11:17:37,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/409d2818fd174d6ea2f8bb53ad3e898b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/409d2818fd174d6ea2f8bb53ad3e898b 2024-11-26T11:17:37,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/239d72ad3bbd42feb367acf36c4a4826 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/239d72ad3bbd42feb367acf36c4a4826 2024-11-26T11:17:37,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0827f33f6bf443ebb4eeb5c455736d62 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0827f33f6bf443ebb4eeb5c455736d62 2024-11-26T11:17:37,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1e1716b290fe431e8e134e66dc13b47f to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1e1716b290fe431e8e134e66dc13b47f 2024-11-26T11:17:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/c758c03342084b1ba69b6e1f9254e259 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/c758c03342084b1ba69b6e1f9254e259 2024-11-26T11:17:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/4583de94b37843e2b305e700ae083cef to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/4583de94b37843e2b305e700ae083cef 2024-11-26T11:17:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/137cd646406b42c28d1681b0f7fe436c to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/137cd646406b42c28d1681b0f7fe436c 2024-11-26T11:17:37,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0cdcd7b2e0624932b286a7f4c14be3cc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/0cdcd7b2e0624932b286a7f4c14be3cc 2024-11-26T11:17:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1789c6199d4941679e2aa2430bb8b58e to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/1789c6199d4941679e2aa2430bb8b58e 2024-11-26T11:17:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/079842749cff42bea7b344bdff8d4151 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/079842749cff42bea7b344bdff8d4151 2024-11-26T11:17:37,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/08d18551510b4d7490aa94b902d483c3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/08d18551510b4d7490aa94b902d483c3 2024-11-26T11:17:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/af6e905867ef42d88a94c8a9986c7e9d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/af6e905867ef42d88a94c8a9986c7e9d 2024-11-26T11:17:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/83813c9b862242448bbdfbaad2d60a43 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/83813c9b862242448bbdfbaad2d60a43 2024-11-26T11:17:37,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/945d641ce9754e7e9502bbf7658211e1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/945d641ce9754e7e9502bbf7658211e1 2024-11-26T11:17:37,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ab0484a4a49b435881da4e23f907f061 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ab0484a4a49b435881da4e23f907f061 2024-11-26T11:17:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/bbd3c3e783ea41139fa39225f00b55eb to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/bbd3c3e783ea41139fa39225f00b55eb 2024-11-26T11:17:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/6d09695c4b7e486aab9d3610f92c5670 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/6d09695c4b7e486aab9d3610f92c5670 2024-11-26T11:17:37,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/b7cee3e571394c2482557adb67b7b6b6 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/b7cee3e571394c2482557adb67b7b6b6 2024-11-26T11:17:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/eca3ffb837d743b491d84425daf7dd90 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/eca3ffb837d743b491d84425daf7dd90 2024-11-26T11:17:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/8d03c9d58a8f4775a3bf036b9cc3f494 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/8d03c9d58a8f4775a3bf036b9cc3f494 2024-11-26T11:17:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e8973dd6f62b4f8ab946538b888f4a67 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e8973dd6f62b4f8ab946538b888f4a67 2024-11-26T11:17:37,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e898ae35b1cb4ed2981e436f56cc860b to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/e898ae35b1cb4ed2981e436f56cc860b 2024-11-26T11:17:37,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/67d132aee3c0425a8ecd3fa45691fdd2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/67d132aee3c0425a8ecd3fa45691fdd2 2024-11-26T11:17:37,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/aafe370facdf40edafe4d9aede4ac7a5 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/aafe370facdf40edafe4d9aede4ac7a5 2024-11-26T11:17:37,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ff25b07c7b494f2787309fc77428b677 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/ff25b07c7b494f2787309fc77428b677 2024-11-26T11:17:37,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/311d498d93294ba0883d64d480c1f3e8 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/311d498d93294ba0883d64d480c1f3e8 2024-11-26T11:17:37,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/32388a2d00324c56ada47cc15c7b6374, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/0498c39621094c9baf278d705a51d0f1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/499d6d42961c445aa43b6125db229bd2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b18f6b0dbb0945aea0f2618a15553df3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/758c94bbfabf4281beedfbc8ba5f69c1, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/514f35521f3e4273b073fccdefe54b5a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/295f48eb9a034f1889a98e45669689a2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/2c3c8d9e3bf54c51a4f7c319d9e92e38, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/33dc7e61eb004b428f60302c55579172, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/dec5974467d54abc884a67bcec2c5337, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/66127a572d074637bee730c08e08af1d, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/70b9733e27884bd8911bfc82dc6ebe62, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1348b9ed87484f7293038c6acb98ff26, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/71a6877770b142699e446d1daad42dc2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/21b0bf501a5f422d8ca24472f76b5031, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/8f2954257fde433999930566bae7ffbf, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1f046dbdef9947a7adf8ea179be75567, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/3f580d91184e4c96960ce4130d49a9c2, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/c4fc8c8928cf48d79df235c417bc5b03, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/327604a2f7f84bfaacbbae9a3dd8da02, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/9a03758cb3ac4bf495f72250ab7f4443, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/ee783a6fc0074cc6bd97d7aca6f91de3, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/776538b770cd4bc49b9b00b1e8a5f86a, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/f5c4a53547924fefa119d35266e82135, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b86a8cbee0184f839c1a49365cbd7527, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/96104f97ed814a439030f925b4522052] to archive 2024-11-26T11:17:37,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T11:17:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/32388a2d00324c56ada47cc15c7b6374 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/32388a2d00324c56ada47cc15c7b6374 2024-11-26T11:17:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/0498c39621094c9baf278d705a51d0f1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/0498c39621094c9baf278d705a51d0f1 2024-11-26T11:17:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/d4e61cbf21cd4f8592fe67cfb1dc9bf3 2024-11-26T11:17:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/499d6d42961c445aa43b6125db229bd2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/499d6d42961c445aa43b6125db229bd2 2024-11-26T11:17:37,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b18f6b0dbb0945aea0f2618a15553df3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b18f6b0dbb0945aea0f2618a15553df3 2024-11-26T11:17:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/758c94bbfabf4281beedfbc8ba5f69c1 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/758c94bbfabf4281beedfbc8ba5f69c1 2024-11-26T11:17:37,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/514f35521f3e4273b073fccdefe54b5a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/514f35521f3e4273b073fccdefe54b5a 2024-11-26T11:17:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/295f48eb9a034f1889a98e45669689a2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/295f48eb9a034f1889a98e45669689a2 2024-11-26T11:17:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/2c3c8d9e3bf54c51a4f7c319d9e92e38 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/2c3c8d9e3bf54c51a4f7c319d9e92e38 2024-11-26T11:17:37,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/33dc7e61eb004b428f60302c55579172 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/33dc7e61eb004b428f60302c55579172 2024-11-26T11:17:37,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/dec5974467d54abc884a67bcec2c5337 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/dec5974467d54abc884a67bcec2c5337 2024-11-26T11:17:37,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/66127a572d074637bee730c08e08af1d to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/66127a572d074637bee730c08e08af1d 2024-11-26T11:17:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/70b9733e27884bd8911bfc82dc6ebe62 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/70b9733e27884bd8911bfc82dc6ebe62 2024-11-26T11:17:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1348b9ed87484f7293038c6acb98ff26 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1348b9ed87484f7293038c6acb98ff26 2024-11-26T11:17:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/71a6877770b142699e446d1daad42dc2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/71a6877770b142699e446d1daad42dc2 2024-11-26T11:17:37,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/21b0bf501a5f422d8ca24472f76b5031 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/21b0bf501a5f422d8ca24472f76b5031 2024-11-26T11:17:37,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/8f2954257fde433999930566bae7ffbf to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/8f2954257fde433999930566bae7ffbf 2024-11-26T11:17:37,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1f046dbdef9947a7adf8ea179be75567 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/1f046dbdef9947a7adf8ea179be75567 2024-11-26T11:17:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/3f580d91184e4c96960ce4130d49a9c2 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/3f580d91184e4c96960ce4130d49a9c2 2024-11-26T11:17:37,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/c4fc8c8928cf48d79df235c417bc5b03 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/c4fc8c8928cf48d79df235c417bc5b03 2024-11-26T11:17:37,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/327604a2f7f84bfaacbbae9a3dd8da02 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/327604a2f7f84bfaacbbae9a3dd8da02 2024-11-26T11:17:37,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/9a03758cb3ac4bf495f72250ab7f4443 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/9a03758cb3ac4bf495f72250ab7f4443 2024-11-26T11:17:37,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/ee783a6fc0074cc6bd97d7aca6f91de3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/ee783a6fc0074cc6bd97d7aca6f91de3 2024-11-26T11:17:37,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/776538b770cd4bc49b9b00b1e8a5f86a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/776538b770cd4bc49b9b00b1e8a5f86a 2024-11-26T11:17:37,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/f5c4a53547924fefa119d35266e82135 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/f5c4a53547924fefa119d35266e82135 2024-11-26T11:17:37,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b86a8cbee0184f839c1a49365cbd7527 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/b86a8cbee0184f839c1a49365cbd7527 2024-11-26T11:17:37,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/96104f97ed814a439030f925b4522052 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/96104f97ed814a439030f925b4522052 2024-11-26T11:17:37,057 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/recovered.edits/510.seqid, newMaxSeqId=510, maxSeqId=4 2024-11-26T11:17:37,057 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. 2024-11-26T11:17:37,057 DEBUG [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for fa0c0e175d4b592132f0942e2a244b29: Waiting for close lock at 1732619855286Running coprocessor pre-close hooks at 1732619855286Disabling compacts and flushes for region at 1732619855286Disabling writes for close at 1732619855286Obtaining lock to block concurrent updates at 1732619855286Preparing flush snapshotting stores in fa0c0e175d4b592132f0942e2a244b29 at 1732619855286Finished memstore snapshotting TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732619855287 (+1 ms)Flushing stores of TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29. at 1732619855287Flushing fa0c0e175d4b592132f0942e2a244b29/A: creating writer at 1732619855287Flushing mob file fa0c0e175d4b592132f0942e2a244b29/A: appending metadata at 1732619855294 (+7 ms)Flushing mob file fa0c0e175d4b592132f0942e2a244b29/A: closing flushed file at 1732619855294Flushing fa0c0e175d4b592132f0942e2a244b29/A: appending metadata at 1732619855711 (+417 ms)Flushing fa0c0e175d4b592132f0942e2a244b29/A: closing flushed file at 1732619855711Flushing fa0c0e175d4b592132f0942e2a244b29/B: creating writer at 1732619856125 (+414 ms)Flushing fa0c0e175d4b592132f0942e2a244b29/B: appending metadata at 1732619856128 (+3 ms)Flushing fa0c0e175d4b592132f0942e2a244b29/B: closing flushed file at 1732619856128Flushing fa0c0e175d4b592132f0942e2a244b29/C: creating writer at 1732619856545 (+417 ms)Flushing fa0c0e175d4b592132f0942e2a244b29/C: appending metadata at 1732619856548 (+3 ms)Flushing fa0c0e175d4b592132f0942e2a244b29/C: closing flushed file at 1732619856549 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e52831: reopening flushed file at 1732619856962 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a598537: reopening flushed file at 1732619856966 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b9c277b: reopening flushed file at 1732619856971 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fa0c0e175d4b592132f0942e2a244b29 in 1689ms, sequenceid=507, compaction requested=false at 1732619856976 (+5 ms)Writing region close event to WAL at 1732619857055 (+79 ms)Running coprocessor post-close hooks at 1732619857057 (+2 ms)Closed at 1732619857057 2024-11-26T11:17:37,059 INFO [RS_CLOSE_REGION-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,059 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=fa0c0e175d4b592132f0942e2a244b29, regionState=CLOSED 2024-11-26T11:17:37,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 because future has completed 2024-11-26T11:17:37,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-26T11:17:37,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure fa0c0e175d4b592132f0942e2a244b29, server=7f3a3edaecc3,37499,1732619676667 in 1.9270 sec 2024-11-26T11:17:37,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-26T11:17:37,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa0c0e175d4b592132f0942e2a244b29, UNASSIGN in 1.9320 sec 2024-11-26T11:17:37,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-26T11:17:37,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9350 sec 2024-11-26T11:17:37,065 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732619857065"}]},"ts":"1732619857065"} 2024-11-26T11:17:37,066 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T11:17:37,066 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T11:17:37,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9470 sec 2024-11-26T11:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T11:17:37,283 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:37,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T11:17:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:37,288 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:37,289 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T11:17:37,291 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C, FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/recovered.edits] 2024-11-26T11:17:37,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/26d58a23f4ec4aceb3198584136a2aa4 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/26d58a23f4ec4aceb3198584136a2aa4 2024-11-26T11:17:37,298 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d651be8833384ff9bc8bdcde9c44ab6a to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/A/d651be8833384ff9bc8bdcde9c44ab6a 2024-11-26T11:17:37,301 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/023c9e6e9d21453789e788aac7237a44 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/023c9e6e9d21453789e788aac7237a44 2024-11-26T11:17:37,302 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/862f65fd93a7493b86208163034aabbc to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/B/862f65fd93a7493b86208163034aabbc 2024-11-26T11:17:37,305 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/adb3d53868dd4c78aba87436767572e3 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/adb3d53868dd4c78aba87436767572e3 2024-11-26T11:17:37,306 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/e0e932609b32414b89a9162896d8cb70 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/C/e0e932609b32414b89a9162896d8cb70 2024-11-26T11:17:37,309 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/recovered.edits/510.seqid to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29/recovered.edits/510.seqid 2024-11-26T11:17:37,309 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/default/TestAcidGuarantees/fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,309 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T11:17:37,310 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T11:17:37,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T11:17:37,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126026e9a56934a4edd8b94e77541f586ce_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126026e9a56934a4edd8b94e77541f586ce_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,316 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e41a978571e4eb5b2bd1dbf793cba1c_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260e41a978571e4eb5b2bd1dbf793cba1c_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,317 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112617bc2e54fd4f4a858828472bfe012c86_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112617bc2e54fd4f4a858828472bfe012c86_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e8030af222d4481aec5bec62811ccc4_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e8030af222d4481aec5bec62811ccc4_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e81fb5e830e446ba3f1cf9c4aeeb71a_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261e81fb5e830e446ba3f1cf9c4aeeb71a_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,319 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653b044cf74eb46609db96bf16225ed01_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653b044cf74eb46609db96bf16225ed01_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a750d523ae147db9d3d77f9053c4e22_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a750d523ae147db9d3d77f9053c4e22_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265d6acd0bef54491eb823b1cf563ff51a_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265d6acd0bef54491eb823b1cf563ff51a_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,321 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266093dec584c34cdd87730b49b2082a0b_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266093dec584c34cdd87730b49b2082a0b_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666d86c305dad4629a96c418de3d7a274_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666d86c305dad4629a96c418de3d7a274_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266c8eece0f96b4fc3bbb847f07ee44524_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266c8eece0f96b4fc3bbb847f07ee44524_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,323 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112678bf00c9917a4668bd0ff5cb8a9f0ba2_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112678bf00c9917a4668bd0ff5cb8a9f0ba2_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,324 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112698600fd7af09479bb55ad06859134ec9_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112698600fd7af09479bb55ad06859134ec9_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,324 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126afd5052d157440b2a37b820912cd6ae0_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126afd5052d157440b2a37b820912cd6ae0_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,325 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ba69165a24ed4010a5f592bf4a195bbd_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ba69165a24ed4010a5f592bf4a195bbd_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,326 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c606f8d8b68a4bb7914460db24954ef2_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c606f8d8b68a4bb7914460db24954ef2_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,326 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c90ff0a955ab4fd0b10eece7b36a8138_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c90ff0a955ab4fd0b10eece7b36a8138_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,327 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dbda699a3940402f8b3acc0a1573fbf3_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dbda699a3940402f8b3acc0a1573fbf3_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,328 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e282f1d22f3a44ab860a0ff60e1a5622_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e282f1d22f3a44ab860a0ff60e1a5622_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,329 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e7e4e16031ef4fa98aab2cdec1f21334_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e7e4e16031ef4fa98aab2cdec1f21334_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,330 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f38e6d87379242ca8ba14da344ee678d_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f38e6d87379242ca8ba14da344ee678d_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,331 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ff1bb2e82f044e0091d5b2b80b703aa1_fa0c0e175d4b592132f0942e2a244b29 to hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ff1bb2e82f044e0091d5b2b80b703aa1_fa0c0e175d4b592132f0942e2a244b29 2024-11-26T11:17:37,332 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T11:17:37,334 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:37,337 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T11:17:37,339 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T11:17:37,340 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:37,341 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T11:17:37,341 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732619857341"}]},"ts":"9223372036854775807"} 2024-11-26T11:17:37,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T11:17:37,343 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fa0c0e175d4b592132f0942e2a244b29, NAME => 'TestAcidGuarantees,,1732619825924.fa0c0e175d4b592132f0942e2a244b29.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T11:17:37,343 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T11:17:37,343 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732619857343"}]},"ts":"9223372036854775807"} 2024-11-26T11:17:37,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T11:17:37,345 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T11:17:37,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-26T11:17:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42255 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-26T11:17:37,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T11:17:37,394 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T11:17:37,402 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 421) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6797 (was 2681) - AvailableMemoryMB LEAK? - 2024-11-26T11:17:37,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T11:17:37,403 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:17:37,403 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:17:37,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:37,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:37,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T11:17:37,403 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T11:17:37,403 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=164229068, stopped=false 2024-11-26T11:17:37,403 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7f3a3edaecc3,42255,1732619675105 2024-11-26T11:17:37,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T11:17:37,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T11:17:37,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:17:37,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:17:37,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T11:17:37,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T11:17:37,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T11:17:37,517 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T11:17:37,517 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:17:37,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:37,518 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7f3a3edaecc3,37499,1732619676667' ***** 2024-11-26T11:17:37,518 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T11:17:37,519 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T11:17:37,519 INFO [RS:0;7f3a3edaecc3:37499 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T11:17:37,519 INFO [RS:0;7f3a3edaecc3:37499 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T11:17:37,519 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T11:17:37,519 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(959): stopping server 7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:37,519 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T11:17:37,519 INFO [RS:0;7f3a3edaecc3:37499 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7f3a3edaecc3:37499. 2024-11-26T11:17:37,520 DEBUG [RS:0;7f3a3edaecc3:37499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T11:17:37,520 DEBUG [RS:0;7f3a3edaecc3:37499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:37,520 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T11:17:37,520 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T11:17:37,520 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T11:17:37,520 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T11:17:37,521 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T11:17:37,521 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T11:17:37,521 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:37,521 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T11:17:37,521 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T11:17:37,521 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T11:17:37,521 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T11:17:37,521 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T11:17:37,521 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T11:17:37,544 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/info/9a254946fcec46048ce2e7a28e418795 is 95, key is TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad./info:/1732619708114/DeleteFamily/seqid=0 2024-11-26T11:17:37,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742444_1620 (size=6514) 2024-11-26T11:17:37,578 INFO [regionserver/7f3a3edaecc3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T11:17:37,721 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:37,812 INFO [regionserver/7f3a3edaecc3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T11:17:37,812 INFO [regionserver/7f3a3edaecc3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T11:17:37,921 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:37,948 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/info/9a254946fcec46048ce2e7a28e418795 2024-11-26T11:17:37,968 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/ns/81ef30d06a68481b9774b56ac4d15e39 is 93, key is TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad./ns:/1732619708064/DeleteFamily/seqid=0 2024-11-26T11:17:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742445_1621 (size=5906) 2024-11-26T11:17:38,122 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:38,322 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:38,373 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/ns/81ef30d06a68481b9774b56ac4d15e39 2024-11-26T11:17:38,401 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/rep_barrier/5e4e076163ee40f98b6638e331198f09 is 102, key is TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad./rep_barrier:/1732619708064/DeleteFamily/seqid=0 2024-11-26T11:17:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742446_1622 (size=6025) 2024-11-26T11:17:38,522 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:38,723 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:38,805 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/rep_barrier/5e4e076163ee40f98b6638e331198f09 2024-11-26T11:17:38,830 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/table/912f46124092444181f7e23dde1011c8 is 96, key is TestAcidGuarantees,,1732619681437.77c23a8be60706f6017524015b3171ad./table:/1732619708064/DeleteFamily/seqid=0 2024-11-26T11:17:38,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742447_1623 (size=5936) 2024-11-26T11:17:38,923 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:39,123 DEBUG [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T11:17:39,234 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/table/912f46124092444181f7e23dde1011c8 2024-11-26T11:17:39,238 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/info/9a254946fcec46048ce2e7a28e418795 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/info/9a254946fcec46048ce2e7a28e418795 2024-11-26T11:17:39,242 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/info/9a254946fcec46048ce2e7a28e418795, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T11:17:39,243 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/ns/81ef30d06a68481b9774b56ac4d15e39 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/ns/81ef30d06a68481b9774b56ac4d15e39 2024-11-26T11:17:39,246 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/ns/81ef30d06a68481b9774b56ac4d15e39, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T11:17:39,247 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/rep_barrier/5e4e076163ee40f98b6638e331198f09 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/rep_barrier/5e4e076163ee40f98b6638e331198f09 2024-11-26T11:17:39,250 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/rep_barrier/5e4e076163ee40f98b6638e331198f09, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T11:17:39,250 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/.tmp/table/912f46124092444181f7e23dde1011c8 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/table/912f46124092444181f7e23dde1011c8 2024-11-26T11:17:39,253 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/table/912f46124092444181f7e23dde1011c8, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T11:17:39,254 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1733ms, sequenceid=90, compaction requested=false 2024-11-26T11:17:39,259 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T11:17:39,260 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T11:17:39,260 INFO [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T11:17:39,260 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732619857521Running coprocessor pre-close hooks at 1732619857521Disabling compacts and flushes for region at 1732619857521Disabling writes for close at 1732619857521Obtaining lock to block concurrent updates at 1732619857521Preparing flush snapshotting stores in 1588230740 at 1732619857521Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732619857522 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732619857522Flushing 1588230740/info: creating writer at 1732619857522Flushing 1588230740/info: appending metadata at 1732619857543 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732619857543Flushing 1588230740/ns: creating writer at 1732619857952 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732619857968 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732619857968Flushing 1588230740/rep_barrier: creating writer at 1732619858382 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732619858401 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732619858401Flushing 1588230740/table: creating writer at 1732619858812 (+411 ms)Flushing 1588230740/table: appending metadata at 1732619858830 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732619858830Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fd4209b: reopening flushed file at 1732619859238 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb27935: reopening flushed file at 1732619859243 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fd22280: reopening flushed file at 1732619859247 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6272f58b: reopening flushed file at 1732619859250 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1733ms, sequenceid=90, compaction requested=false at 1732619859254 (+4 ms)Writing region close event to WAL at 1732619859256 (+2 ms)Running coprocessor post-close hooks at 1732619859259 (+3 ms)Closed at 1732619859260 (+1 ms) 2024-11-26T11:17:39,260 DEBUG [RS_CLOSE_META-regionserver/7f3a3edaecc3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T11:17:39,324 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(976): stopping server 7f3a3edaecc3,37499,1732619676667; all regions closed. 2024-11-26T11:17:39,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741834_1010 (size=25344) 2024-11-26T11:17:39,333 DEBUG [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/oldWALs 2024-11-26T11:17:39,333 INFO [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7f3a3edaecc3%2C37499%2C1732619676667.meta:.meta(num 1732619680988) 2024-11-26T11:17:39,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741832_1008 (size=19428564) 2024-11-26T11:17:39,339 DEBUG [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/oldWALs 2024-11-26T11:17:39,339 INFO [RS:0;7f3a3edaecc3:37499 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7f3a3edaecc3%2C37499%2C1732619676667:(num 1732619679972) 2024-11-26T11:17:39,339 DEBUG [RS:0;7f3a3edaecc3:37499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T11:17:39,339 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T11:17:39,339 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T11:17:39,339 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.ChoreService(370): Chore service for: regionserver/7f3a3edaecc3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T11:17:39,339 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T11:17:39,340 INFO [regionserver/7f3a3edaecc3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T11:17:39,341 INFO [RS:0;7f3a3edaecc3:37499 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37499 2024-11-26T11:17:39,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7f3a3edaecc3,37499,1732619676667 2024-11-26T11:17:39,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T11:17:39,355 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T11:17:39,357 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7f3a3edaecc3,37499,1732619676667] 2024-11-26T11:17:39,376 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7f3a3edaecc3,37499,1732619676667 already deleted, retry=false 2024-11-26T11:17:39,376 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7f3a3edaecc3,37499,1732619676667 expired; onlineServers=0 2024-11-26T11:17:39,376 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7f3a3edaecc3,42255,1732619675105' ***** 2024-11-26T11:17:39,377 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T11:17:39,377 INFO [M:0;7f3a3edaecc3:42255 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T11:17:39,377 INFO [M:0;7f3a3edaecc3:42255 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T11:17:39,377 DEBUG [M:0;7f3a3edaecc3:42255 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T11:17:39,377 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T11:17:39,377 DEBUG [M:0;7f3a3edaecc3:42255 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T11:17:39,377 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster-HFileCleaner.small.0-1732619679652 {}] cleaner.HFileCleaner(306): Exit Thread[master/7f3a3edaecc3:0:becomeActiveMaster-HFileCleaner.small.0-1732619679652,5,FailOnTimeoutGroup] 2024-11-26T11:17:39,377 DEBUG [master/7f3a3edaecc3:0:becomeActiveMaster-HFileCleaner.large.0-1732619679650 {}] cleaner.HFileCleaner(306): Exit Thread[master/7f3a3edaecc3:0:becomeActiveMaster-HFileCleaner.large.0-1732619679650,5,FailOnTimeoutGroup] 2024-11-26T11:17:39,377 INFO [M:0;7f3a3edaecc3:42255 {}] hbase.ChoreService(370): Chore service for: master/7f3a3edaecc3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T11:17:39,377 INFO [M:0;7f3a3edaecc3:42255 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T11:17:39,377 DEBUG [M:0;7f3a3edaecc3:42255 {}] master.HMaster(1795): Stopping service threads 2024-11-26T11:17:39,378 INFO [M:0;7f3a3edaecc3:42255 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T11:17:39,378 INFO [M:0;7f3a3edaecc3:42255 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T11:17:39,378 ERROR [M:0;7f3a3edaecc3:42255 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T11:17:39,378 INFO [M:0;7f3a3edaecc3:42255 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T11:17:39,378 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T11:17:39,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T11:17:39,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T11:17:39,387 DEBUG [M:0;7f3a3edaecc3:42255 {}] zookeeper.ZKUtil(347): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T11:17:39,387 WARN [M:0;7f3a3edaecc3:42255 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T11:17:39,388 INFO [M:0;7f3a3edaecc3:42255 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/.lastflushedseqids 2024-11-26T11:17:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742448_1624 (size=72) 2024-11-26T11:17:39,402 INFO [M:0;7f3a3edaecc3:42255 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T11:17:39,402 INFO [M:0;7f3a3edaecc3:42255 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T11:17:39,402 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T11:17:39,402 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:17:39,402 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:17:39,402 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T11:17:39,402 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:17:39,402 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=760.69 KB heapSize=930.41 KB 2024-11-26T11:17:39,419 DEBUG [M:0;7f3a3edaecc3:42255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9801681745454948ab7d580cd8c031e1 is 82, key is hbase:meta,,1/info:regioninfo/1732619681070/Put/seqid=0 2024-11-26T11:17:39,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742449_1625 (size=5672) 2024-11-26T11:17:39,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T11:17:39,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37499-0x10176da1e780001, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T11:17:39,467 INFO [RS:0;7f3a3edaecc3:37499 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T11:17:39,467 INFO [RS:0;7f3a3edaecc3:37499 {}] regionserver.HRegionServer(1031): Exiting; stopping=7f3a3edaecc3,37499,1732619676667; zookeeper connection closed. 2024-11-26T11:17:39,467 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5bf3daf3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5bf3daf3 2024-11-26T11:17:39,468 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T11:17:39,823 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2094 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9801681745454948ab7d580cd8c031e1 2024-11-26T11:17:39,845 DEBUG [M:0;7f3a3edaecc3:42255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0323244b3ef452e85a6af48c9b9f03a is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x87/proc:d/1732619829026/Put/seqid=0 2024-11-26T11:17:39,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742450_1626 (size=44384) 2024-11-26T11:17:40,248 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=760.13 KB at sequenceid=2094 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0323244b3ef452e85a6af48c9b9f03a 2024-11-26T11:17:40,252 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0323244b3ef452e85a6af48c9b9f03a 2024-11-26T11:17:40,266 DEBUG [M:0;7f3a3edaecc3:42255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3dbfa441f26249939dd4b78ea2650b1b is 69, key is 7f3a3edaecc3,37499,1732619676667/rs:state/1732619679723/Put/seqid=0 2024-11-26T11:17:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073742451_1627 (size=5156) 2024-11-26T11:17:40,670 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2094 (bloomFilter=true), to=hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3dbfa441f26249939dd4b78ea2650b1b 2024-11-26T11:17:40,674 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9801681745454948ab7d580cd8c031e1 as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9801681745454948ab7d580cd8c031e1 2024-11-26T11:17:40,677 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9801681745454948ab7d580cd8c031e1, entries=8, sequenceid=2094, filesize=5.5 K 2024-11-26T11:17:40,678 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0323244b3ef452e85a6af48c9b9f03a as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0323244b3ef452e85a6af48c9b9f03a 2024-11-26T11:17:40,681 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0323244b3ef452e85a6af48c9b9f03a 2024-11-26T11:17:40,681 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0323244b3ef452e85a6af48c9b9f03a, entries=164, sequenceid=2094, filesize=43.3 K 2024-11-26T11:17:40,682 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3dbfa441f26249939dd4b78ea2650b1b as hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3dbfa441f26249939dd4b78ea2650b1b 2024-11-26T11:17:40,686 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41651/user/jenkins/test-data/4be95c84-41b6-dec3-953b-54995dd7ed29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3dbfa441f26249939dd4b78ea2650b1b, entries=1, sequenceid=2094, filesize=5.0 K 2024-11-26T11:17:40,687 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(3140): Finished flush of dataSize ~760.69 KB/778945, heapSize ~930.11 KB/952432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2094, compaction requested=false 2024-11-26T11:17:40,695 INFO [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T11:17:40,695 DEBUG [M:0;7f3a3edaecc3:42255 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732619859402Disabling compacts and flushes for region at 1732619859402Disabling writes for close at 1732619859402Obtaining lock to block concurrent updates at 1732619859402Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732619859402Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=778945, getHeapSize=952672, getOffHeapSize=0, getCellsCount=2283 at 1732619859403 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732619859403Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732619859403Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732619859418 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732619859418Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732619859826 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732619859844 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732619859844Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732619860252 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732619860266 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732619860266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44e79ed9: reopening flushed file at 1732619860673 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a5c3d63: reopening flushed file at 1732619860677 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@457bb67b: reopening flushed file at 1732619860681 (+4 ms)Finished flush of dataSize ~760.69 KB/778945, heapSize ~930.11 KB/952432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2094, compaction requested=false at 1732619860687 (+6 ms)Writing region close event to WAL at 1732619860695 (+8 ms)Closed at 1732619860695 2024-11-26T11:17:40,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41003 is added to blk_1073741830_1006 (size=916230) 2024-11-26T11:17:40,699 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T11:17:40,699 INFO [M:0;7f3a3edaecc3:42255 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T11:17:40,700 INFO [M:0;7f3a3edaecc3:42255 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42255 2024-11-26T11:17:40,700 INFO [M:0;7f3a3edaecc3:42255 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T11:17:40,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T11:17:40,845 INFO [M:0;7f3a3edaecc3:42255 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T11:17:40,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42255-0x10176da1e780000, quorum=127.0.0.1:64486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T11:17:40,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T11:17:40,853 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T11:17:40,853 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T11:17:40,854 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T11:17:40,854 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/hadoop.log.dir/,STOPPED} 2024-11-26T11:17:40,857 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T11:17:40,857 WARN [BP-1195852732-172.17.0.3-1732619671074 heartbeating to localhost/127.0.0.1:41651 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T11:17:40,857 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T11:17:40,857 WARN [BP-1195852732-172.17.0.3-1732619671074 heartbeating to localhost/127.0.0.1:41651 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1195852732-172.17.0.3-1732619671074 (Datanode Uuid 64bbef44-b7bc-4876-9241-855d1bf347e8) service to localhost/127.0.0.1:41651 2024-11-26T11:17:40,861 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/data/data1/current/BP-1195852732-172.17.0.3-1732619671074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T11:17:40,862 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/cluster_84aea61d-3f78-d134-c7a5-14f2fb6a2ce9/data/data2/current/BP-1195852732-172.17.0.3-1732619671074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T11:17:40,862 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T11:17:40,869 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T11:17:40,870 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T11:17:40,870 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T11:17:40,870 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T11:17:40,870 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/25cc0fbc-2d5a-c6cc-8427-0b9360e33fb8/hadoop.log.dir/,STOPPED} 2024-11-26T11:17:40,885 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T11:17:41,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down